1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  22 /*        All Rights Reserved   */
  23 
  24 
  25 /*
  26  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  27  * Use is subject to license terms.
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/sysmacros.h>
  32 #include <sys/param.h>
  33 #include <sys/errno.h>
  34 #include <sys/signal.h>
  35 #include <sys/proc.h>
  36 #include <sys/conf.h>
  37 #include <sys/cred.h>
  38 #include <sys/user.h>
  39 #include <sys/vnode.h>
  40 #include <sys/file.h>
  41 #include <sys/session.h>
  42 #include <sys/stream.h>
  43 #include <sys/strsubr.h>
  44 #include <sys/stropts.h>
  45 #include <sys/poll.h>
  46 #include <sys/systm.h>
  47 #include <sys/cpuvar.h>
  48 #include <sys/uio.h>
  49 #include <sys/cmn_err.h>
  50 #include <sys/priocntl.h>
  51 #include <sys/procset.h>
  52 #include <sys/vmem.h>
  53 #include <sys/bitmap.h>
  54 #include <sys/kmem.h>
  55 #include <sys/siginfo.h>
  56 #include <sys/vtrace.h>
  57 #include <sys/callb.h>
  58 #include <sys/debug.h>
  59 #include <sys/modctl.h>
  60 #include <sys/vmsystm.h>
  61 #include <vm/page.h>
  62 #include <sys/atomic.h>
  63 #include <sys/suntpi.h>
  64 #include <sys/strlog.h>
  65 #include <sys/promif.h>
  66 #include <sys/project.h>
  67 #include <sys/vm.h>
  68 #include <sys/taskq.h>
  69 #include <sys/sunddi.h>
  70 #include <sys/sunldi_impl.h>
  71 #include <sys/strsun.h>
  72 #include <sys/isa_defs.h>
  73 #include <sys/multidata.h>
  74 #include <sys/pattr.h>
  75 #include <sys/strft.h>
  76 #include <sys/fs/snode.h>
  77 #include <sys/zone.h>
  78 #include <sys/open.h>
  79 #include <sys/sunldi.h>
  80 #include <sys/sad.h>
  81 #include <sys/netstack.h>
  82 
  83 #define O_SAMESTR(q)    (((q)->q_next) && \
  84         (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
  85 
  86 /*
  87  * WARNING:
  88  * The variables and routines in this file are private, belonging
  89  * to the STREAMS subsystem. These should not be used by modules
  90  * or drivers. Compatibility will not be guaranteed.
  91  */
  92 
  93 /*
  94  * Id value used to distinguish between different multiplexor links.
  95  */
  96 static int32_t lnk_id = 0;
  97 
  98 #define STREAMS_LOPRI MINCLSYSPRI
  99 static pri_t streams_lopri = STREAMS_LOPRI;
 100 
 101 #define STRSTAT(x)      (str_statistics.x.value.ui64++)
 102 typedef struct str_stat {
 103         kstat_named_t   sqenables;
 104         kstat_named_t   stenables;
 105         kstat_named_t   syncqservice;
 106         kstat_named_t   freebs;
 107         kstat_named_t   qwr_outer;
 108         kstat_named_t   rservice;
 109         kstat_named_t   strwaits;
 110         kstat_named_t   taskqfails;
 111         kstat_named_t   bufcalls;
 112         kstat_named_t   qhelps;
 113         kstat_named_t   qremoved;
 114         kstat_named_t   sqremoved;
 115         kstat_named_t   bcwaits;
 116         kstat_named_t   sqtoomany;
 117 } str_stat_t;
 118 
 119 static str_stat_t str_statistics = {
 120         { "sqenables",          KSTAT_DATA_UINT64 },
 121         { "stenables",          KSTAT_DATA_UINT64 },
 122         { "syncqservice",       KSTAT_DATA_UINT64 },
 123         { "freebs",             KSTAT_DATA_UINT64 },
 124         { "qwr_outer",          KSTAT_DATA_UINT64 },
 125         { "rservice",           KSTAT_DATA_UINT64 },
 126         { "strwaits",           KSTAT_DATA_UINT64 },
 127         { "taskqfails",         KSTAT_DATA_UINT64 },
 128         { "bufcalls",           KSTAT_DATA_UINT64 },
 129         { "qhelps",             KSTAT_DATA_UINT64 },
 130         { "qremoved",           KSTAT_DATA_UINT64 },
 131         { "sqremoved",          KSTAT_DATA_UINT64 },
 132         { "bcwaits",            KSTAT_DATA_UINT64 },
 133         { "sqtoomany",          KSTAT_DATA_UINT64 },
 134 };
 135 
 136 static kstat_t *str_kstat;
 137 
 138 /*
 139  * qrunflag was used previously to control background scheduling of queues. It
 140  * is not used anymore, but kept here in case some module still wants to access
 141  * it via qready() and setqsched macros.
 142  */
 143 char qrunflag;                  /*  Unused */
 144 
 145 /*
 146  * Most of the streams scheduling is done via task queues. Task queues may fail
 147  * for non-sleep dispatches, so there are two backup threads servicing failed
 148  * requests for queues and syncqs. Both of these threads also service failed
 149  * dispatches freebs requests. Queues are put in the list specified by `qhead'
 150  * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
 151  * requests are put into `freebs_list' which has no tail pointer. All three
 152  * lists are protected by a single `service_queue' lock and use
 153  * `services_to_run' condition variable for signaling background threads. Use of
 154  * a single lock should not be a problem because it is only used under heavy
 155  * loads when task queues start to fail and at that time it may be a good idea
 156  * to throttle scheduling requests.
 157  *
 158  * NOTE: queues and syncqs should be scheduled by two separate threads because
 159  * queue servicing may be blocked waiting for a syncq which may be also
 160  * scheduled for background execution. This may create a deadlock when only one
 161  * thread is used for both.
 162  */
 163 
 164 static taskq_t *streams_taskq;          /* Used for most STREAMS scheduling */
 165 
 166 static kmutex_t service_queue;          /* protects all of servicing vars */
 167 static kcondvar_t services_to_run;      /* wake up background service thread */
 168 static kcondvar_t syncqs_to_run;        /* wake up background service thread */
 169 
 170 /*
 171  * List of queues scheduled for background processing due to lack of resources
 172  * in the task queues. Protected by service_queue lock;
 173  */
 174 static struct queue *qhead;
 175 static struct queue *qtail;
 176 
 177 /*
 178  * Same list for syncqs
 179  */
 180 static syncq_t *sqhead;
 181 static syncq_t *sqtail;
 182 
 183 static mblk_t *freebs_list;     /* list of buffers to free */
 184 
 185 /*
 186  * Backup threads for servicing queues and syncqs
 187  */
 188 kthread_t *streams_qbkgrnd_thread;
 189 kthread_t *streams_sqbkgrnd_thread;
 190 
 191 /*
 192  * Bufcalls related variables.
 193  */
 194 struct bclist   strbcalls;      /* list of waiting bufcalls */
 195 kmutex_t        strbcall_lock;  /* protects bufcall list (strbcalls) */
 196 kcondvar_t      strbcall_cv;    /* Signaling when a bufcall is added */
 197 kmutex_t        bcall_monitor;  /* sleep/wakeup style monitor */
 198 kcondvar_t      bcall_cv;       /* wait 'till executing bufcall completes */
 199 kthread_t       *bc_bkgrnd_thread; /* Thread to service bufcall requests */
 200 
 201 kmutex_t        strresources;   /* protects global resources */
 202 kmutex_t        muxifier;       /* single-threads multiplexor creation */
 203 
 204 static void     *str_stack_init(netstackid_t stackid, netstack_t *ns);
 205 static void     str_stack_shutdown(netstackid_t stackid, void *arg);
 206 static void     str_stack_fini(netstackid_t stackid, void *arg);
 207 
 208 /*
 209  * run_queues is no longer used, but is kept in case some 3rd party
 210  * module/driver decides to use it.
 211  */
 212 int run_queues = 0;
 213 
 214 /*
 215  * sq_max_size is the depth of the syncq (in number of messages) before
 216  * qfill_syncq() starts QFULL'ing destination queues. As its primary
 217  * consumer - IP is no longer D_MTPERMOD, but there may be other
 218  * modules/drivers depend on this syncq flow control, we prefer to
 219  * choose a large number as the default value. For potential
 220  * performance gain, this value is tunable in /etc/system.
 221  */
 222 int sq_max_size = 10000;
 223 
 224 /*
 225  * The number of ciputctrl structures per syncq and stream we create when
 226  * needed.
 227  */
 228 int n_ciputctrl;
 229 int max_n_ciputctrl = 16;
 230 /*
 231  * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
 232  */
 233 int min_n_ciputctrl = 2;
 234 
 235 /*
 236  * Per-driver/module syncqs
 237  * ========================
 238  *
 239  * For drivers/modules that use PERMOD or outer syncqs we keep a list of
 240  * perdm structures, new entries being added (and new syncqs allocated) when
 241  * setq() encounters a module/driver with a streamtab that it hasn't seen
 242  * before.
 243  * The reason for this mechanism is that some modules and drivers share a
 244  * common streamtab and it is necessary for those modules and drivers to also
 245  * share a common PERMOD syncq.
 246  *
 247  * perdm_list --> dm_str == streamtab_1
 248  *                dm_sq == syncq_1
 249  *                dm_ref
 250  *                dm_next --> dm_str == streamtab_2
 251  *                            dm_sq == syncq_2
 252  *                            dm_ref
 253  *                            dm_next --> ... NULL
 254  *
 255  * The dm_ref field is incremented for each new driver/module that takes
 256  * a reference to the perdm structure and hence shares the syncq.
 257  * References are held in the fmodsw_impl_t structure for each STREAMS module
 258  * or the dev_impl array (indexed by device major number) for each driver.
 259  *
 260  * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
 261  *                   ^                 ^ ^               ^
 262  *                   |  ______________/  |               |
 263  *                   | /                 |               |
 264  * dev_impl:     ...|x|y|...          module A        module B
 265  *
 266  * When a module/driver is unloaded the reference count is decremented and,
 267  * when it falls to zero, the perdm structure is removed from the list and
 268  * the syncq is freed (see rele_dm()).
 269  */
 270 perdm_t *perdm_list = NULL;
 271 static krwlock_t perdm_rwlock;
 272 cdevsw_impl_t *devimpl;
 273 
 274 extern struct qinit strdata;
 275 extern struct qinit stwdata;
 276 
 277 static void runservice(queue_t *);
 278 static void streams_bufcall_service(void);
 279 static void streams_qbkgrnd_service(void);
 280 static void streams_sqbkgrnd_service(void);
 281 static syncq_t *new_syncq(void);
 282 static void free_syncq(syncq_t *);
 283 static void outer_insert(syncq_t *, syncq_t *);
 284 static void outer_remove(syncq_t *, syncq_t *);
 285 static void write_now(syncq_t *);
 286 static void clr_qfull(queue_t *);
 287 static void runbufcalls(void);
 288 static void sqenable(syncq_t *);
 289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
 290 static void wait_q_syncq(queue_t *);
 291 static void backenable_insertedq(queue_t *);
 292 
 293 static void queue_service(queue_t *);
 294 static void stream_service(stdata_t *);
 295 static void syncq_service(syncq_t *);
 296 static void qwriter_outer_service(syncq_t *);
 297 static void mblk_free(mblk_t *);
 298 #ifdef DEBUG
 299 static int qprocsareon(queue_t *);
 300 #endif
 301 
 302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
 303 static void reset_nfsrv_ptr(queue_t *, queue_t *);
 304 void set_qfull(queue_t *);
 305 
 306 static void sq_run_events(syncq_t *);
 307 static int propagate_syncq(queue_t *);
 308 
 309 static void     blocksq(syncq_t *, ushort_t, int);
 310 static void     unblocksq(syncq_t *, ushort_t, int);
 311 static int      dropsq(syncq_t *, uint16_t);
 312 static void     emptysq(syncq_t *);
 313 static sqlist_t *sqlist_alloc(struct stdata *, int);
 314 static void     sqlist_free(sqlist_t *);
 315 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
 316 static void     sqlist_insert(sqlist_t *, syncq_t *);
 317 static void     sqlist_insertall(sqlist_t *, queue_t *);
 318 
 319 static void     strsetuio(stdata_t *);
 320 
 321 struct kmem_cache *stream_head_cache;
 322 struct kmem_cache *queue_cache;
 323 struct kmem_cache *syncq_cache;
 324 struct kmem_cache *qband_cache;
 325 struct kmem_cache *linkinfo_cache;
 326 struct kmem_cache *ciputctrl_cache = NULL;
 327 
 328 static linkinfo_t *linkinfo_list;
 329 
 330 /* Global esballoc throttling queue */
 331 static esb_queue_t system_esbq;
 332 
 333 /* Array of esballoc throttling queues, of length esbq_nelem */
 334 static esb_queue_t *volatile system_esbq_array;
 335 static int esbq_nelem;
 336 static kmutex_t esbq_lock;
 337 static int esbq_log2_cpus_per_q = 0;
 338 
 339 /* Scale the system_esbq length by setting number of CPUs per queue. */
 340 uint_t esbq_cpus_per_q = 1;
 341 
 342 /*
 343  * esballoc tunable parameters.
 344  */
 345 int             esbq_max_qlen = 0x16;   /* throttled queue length */
 346 clock_t         esbq_timeout = 0x8;     /* timeout to process esb queue */
 347 
 348 /*
 349  * Routines to handle esballoc queueing.
 350  */
 351 static void esballoc_process_queue(esb_queue_t *);
 352 static void esballoc_enqueue_mblk(mblk_t *);
 353 static void esballoc_timer(void *);
 354 static void esballoc_set_timer(esb_queue_t *, clock_t);
 355 static void esballoc_mblk_free(mblk_t *);
 356 
 357 /*
 358  *  Qinit structure and Module_info structures
 359  *      for passthru read and write queues
 360  */
 361 
 362 static void pass_wput(queue_t *, mblk_t *);
 363 static queue_t *link_addpassthru(stdata_t *);
 364 static void link_rempassthru(queue_t *);
 365 
 366 struct  module_info passthru_info = {
 367         0,
 368         "passthru",
 369         0,
 370         INFPSZ,
 371         STRHIGH,
 372         STRLOW
 373 };
 374 
 375 struct  qinit passthru_rinit = {
 376         (int (*)())putnext,
 377         NULL,
 378         NULL,
 379         NULL,
 380         NULL,
 381         &passthru_info,
 382         NULL
 383 };
 384 
 385 struct  qinit passthru_winit = {
 386         (int (*)()) pass_wput,
 387         NULL,
 388         NULL,
 389         NULL,
 390         NULL,
 391         &passthru_info,
 392         NULL
 393 };
 394 
 395 /*
 396  * Verify correctness of list head/tail pointers.
 397  */
 398 #define LISTCHECK(head, tail, link) {                           \
 399         EQUIV(head, tail);                                      \
 400         IMPLY(tail != NULL, tail->link == NULL);             \
 401 }
 402 
 403 /*
 404  * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
 405  * using a `link' field.
 406  */
 407 #define ENQUEUE(el, head, tail, link) {                         \
 408         ASSERT(el->link == NULL);                            \
 409         LISTCHECK(head, tail, link);                            \
 410         if (head == NULL)                                       \
 411                 head = el;                                      \
 412         else                                                    \
 413                 tail->link = el;                             \
 414         tail = el;                                              \
 415 }
 416 
 417 /*
 418  * Dequeue the first element of the list denoted by `head' and `tail' pointers
 419  * using a `link' field and put result into `el'.
 420  */
 421 #define DQ(el, head, tail, link) {                              \
 422         LISTCHECK(head, tail, link);                            \
 423         el = head;                                              \
 424         if (head != NULL) {                                     \
 425                 head = head->link;                           \
 426                 if (head == NULL)                               \
 427                         tail = NULL;                            \
 428                 el->link = NULL;                             \
 429         }                                                       \
 430 }
 431 
 432 /*
 433  * Remove `el' from the list using `chase' and `curr' pointers and return result
 434  * in `succeed'.
 435  */
 436 #define RMQ(el, head, tail, link, chase, curr, succeed) {       \
 437         LISTCHECK(head, tail, link);                            \
 438         chase = NULL;                                           \
 439         succeed = 0;                                            \
 440         for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
 441                 chase = curr;                                   \
 442         if (curr != NULL) {                                     \
 443                 succeed = 1;                                    \
 444                 ASSERT(curr == el);                             \
 445                 if (chase != NULL)                              \
 446                         chase->link = curr->link;         \
 447                 else                                            \
 448                         head = curr->link;                   \
 449                 curr->link = NULL;                           \
 450                 if (curr == tail)                               \
 451                         tail = chase;                           \
 452         }                                                       \
 453         LISTCHECK(head, tail, link);                            \
 454 }
 455 
 456 /* Handling of delayed messages on the inner syncq. */
 457 
 458 /*
 459  * DEBUG versions should use function versions (to simplify tracing) and
 460  * non-DEBUG kernels should use macro versions.
 461  */
 462 
 463 /*
 464  * Put a queue on the syncq list of queues.
 465  * Assumes SQLOCK held.
 466  */
 467 #define SQPUT_Q(sq, qp)                                                 \
 468 {                                                                       \
 469         ASSERT(MUTEX_HELD(SQLOCK(sq)));                                 \
 470         if (!(qp->q_sqflags & Q_SQQUEUED)) {                             \
 471                 /* The queue should not be linked anywhere */           \
 472                 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
 473                 /* Head and tail may only be NULL simultaneously */     \
 474                 EQUIV(sq->sq_head, sq->sq_tail);                  \
 475                 /* Queue may be only enqueued on its syncq */           \
 476                 ASSERT(sq == qp->q_syncq);                           \
 477                 /* Check the correctness of SQ_MESSAGES flag */         \
 478                 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES));     \
 479                 /* Sanity check first/last elements of the list */      \
 480                 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
 481                 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
 482                 /*                                                      \
 483                  * Sanity check of priority field: empty queue should   \
 484                  * have zero priority                                   \
 485                  * and nqueues equal to zero.                           \
 486                  */                                                     \
 487                 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0);              \
 488                 /* Sanity check of sq_nqueues field */                  \
 489                 EQUIV(sq->sq_head, sq->sq_nqueues);                       \
 490                 if (sq->sq_head == NULL) {                           \
 491                         sq->sq_head = sq->sq_tail = qp;                   \
 492                         sq->sq_flags |= SQ_MESSAGES;                 \
 493                 } else if (qp->q_spri == 0) {                                \
 494                         qp->q_sqprev = sq->sq_tail;                       \
 495                         sq->sq_tail->q_sqnext = qp;                       \
 496                         sq->sq_tail = qp;                            \
 497                 } else {                                                \
 498                         /*                                              \
 499                          * Put this queue in priority order: higher     \
 500                          * priority gets closer to the head.            \
 501                          */                                             \
 502                         queue_t **qpp = &sq->sq_tail;                    \
 503                         queue_t *qnext = NULL;                          \
 504                                                                         \
 505                         while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
 506                                 qnext = *qpp;                           \
 507                                 qpp = &(*qpp)->q_sqprev;         \
 508                         }                                               \
 509                         qp->q_sqnext = qnext;                                \
 510                         qp->q_sqprev = *qpp;                         \
 511                         if (*qpp != NULL) {                             \
 512                                 (*qpp)->q_sqnext = qp;                       \
 513                         } else {                                        \
 514                                 sq->sq_head = qp;                    \
 515                                 sq->sq_pri = sq->sq_head->q_spri;      \
 516                         }                                               \
 517                         *qpp = qp;                                      \
 518                 }                                                       \
 519                 qp->q_sqflags |= Q_SQQUEUED;                         \
 520                 qp->q_sqtstamp = ddi_get_lbolt();                    \
 521                 sq->sq_nqueues++;                                    \
 522         }                                                               \
 523 }
 524 
 525 /*
 526  * Remove a queue from the syncq list
 527  * Assumes SQLOCK held.
 528  */
 529 #define SQRM_Q(sq, qp)                                                  \
 530         {                                                               \
 531                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 532                 ASSERT(qp->q_sqflags & Q_SQQUEUED);                      \
 533                 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL);       \
 534                 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0);               \
 535                 /* Check that the queue is actually in the list */      \
 536                 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp);        \
 537                 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp);        \
 538                 ASSERT(sq->sq_nqueues != 0);                         \
 539                 if (qp->q_sqprev == NULL) {                          \
 540                         /* First queue on list, make head q_sqnext */   \
 541                         sq->sq_head = qp->q_sqnext;                       \
 542                 } else {                                                \
 543                         /* Make prev->next == next */                        \
 544                         qp->q_sqprev->q_sqnext = qp->q_sqnext;         \
 545                 }                                                       \
 546                 if (qp->q_sqnext == NULL) {                          \
 547                         /* Last queue on list, make tail sqprev */      \
 548                         sq->sq_tail = qp->q_sqprev;                       \
 549                 } else {                                                \
 550                         /* Make next->prev == prev */                        \
 551                         qp->q_sqnext->q_sqprev = qp->q_sqprev;         \
 552                 }                                                       \
 553                 /* clear out references on this queue */                \
 554                 qp->q_sqprev = qp->q_sqnext = NULL;                       \
 555                 qp->q_sqflags &= ~Q_SQQUEUED;                            \
 556                 /* If there is nothing queued, clear SQ_MESSAGES */     \
 557                 if (sq->sq_head != NULL) {                           \
 558                         sq->sq_pri = sq->sq_head->q_spri;              \
 559                 } else  {                                               \
 560                         sq->sq_flags &= ~SQ_MESSAGES;                    \
 561                         sq->sq_pri = 0;                                      \
 562                 }                                                       \
 563                 sq->sq_nqueues--;                                    \
 564                 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL ||    \
 565                     (sq->sq_flags & SQ_QUEUED) == 0);                    \
 566         }
 567 
 568 /* Hide the definition from the header file. */
 569 #ifdef SQPUT_MP
 570 #undef SQPUT_MP
 571 #endif
 572 
 573 /*
 574  * Put a message on the queue syncq.
 575  * Assumes QLOCK held.
 576  */
 577 #define SQPUT_MP(qp, mp)                                                \
 578         {                                                               \
 579                 ASSERT(MUTEX_HELD(QLOCK(qp)));                          \
 580                 ASSERT(qp->q_sqhead == NULL ||                               \
 581                     (qp->q_sqtail != NULL &&                         \
 582                     qp->q_sqtail->b_next == NULL));                       \
 583                 qp->q_syncqmsgs++;                                   \
 584                 ASSERT(qp->q_syncqmsgs != 0);        /* Wraparound */        \
 585                 if (qp->q_sqhead == NULL) {                          \
 586                         qp->q_sqhead = qp->q_sqtail = mp;         \
 587                 } else {                                                \
 588                         qp->q_sqtail->b_next = mp;                        \
 589                         qp->q_sqtail = mp;                           \
 590                 }                                                       \
 591                 ASSERT(qp->q_syncqmsgs > 0);                              \
 592                 set_qfull(qp);                                          \
 593         }
 594 
 595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) {                                \
 596                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 597                 if ((sq)->sq_ciputctrl != NULL) {                    \
 598                         int i;                                          \
 599                         int nlocks = (sq)->sq_nciputctrl;            \
 600                         ciputctrl_t *cip = (sq)->sq_ciputctrl;               \
 601                         ASSERT((sq)->sq_type & SQ_CIPUT);                \
 602                         for (i = 0; i <= nlocks; i++) {                      \
 603                                 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
 604                                 cip[i].ciputctrl_count |= SQ_FASTPUT;   \
 605                         }                                               \
 606                 }                                                       \
 607         }
 608 
 609 
 610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) {                                \
 611                 ASSERT(MUTEX_HELD(SQLOCK(sq)));                         \
 612                 if ((sq)->sq_ciputctrl != NULL) {                    \
 613                         int i;                                          \
 614                         int nlocks = (sq)->sq_nciputctrl;            \
 615                         ciputctrl_t *cip = (sq)->sq_ciputctrl;               \
 616                         ASSERT((sq)->sq_type & SQ_CIPUT);                \
 617                         for (i = 0; i <= nlocks; i++) {                      \
 618                                 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
 619                                 cip[i].ciputctrl_count &= ~SQ_FASTPUT;      \
 620                         }                                               \
 621                 }                                                       \
 622         }
 623 
 624 /*
 625  * Run service procedures for all queues in the stream head.
 626  */
 627 #define STR_SERVICE(stp, q) {                                           \
 628         ASSERT(MUTEX_HELD(&stp->sd_qlock));                              \
 629         while (stp->sd_qhead != NULL) {                                      \
 630                 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link);              \
 631                 ASSERT(stp->sd_nqueues > 0);                              \
 632                 stp->sd_nqueues--;                                   \
 633                 ASSERT(!(q->q_flag & QINSERVICE));                       \
 634                 mutex_exit(&stp->sd_qlock);                              \
 635                 queue_service(q);                                       \
 636                 mutex_enter(&stp->sd_qlock);                             \
 637         }                                                               \
 638         ASSERT(stp->sd_nqueues == 0);                                        \
 639         ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL));       \
 640 }
 641 
 642 /*
 643  * Constructor/destructor routines for the stream head cache
 644  */
 645 /* ARGSUSED */
 646 static int
 647 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
 648 {
 649         stdata_t *stp = buf;
 650 
 651         mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
 652         mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
 653         mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
 654         mutex_init(&stp->sd_pid_tree_lock, NULL, MUTEX_DEFAULT, NULL);
 655         cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
 656         cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
 657         cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
 658         cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
 659         cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
 660         avl_create(&stp->sd_pid_tree, pid_node_comparator, sizeof (pid_node_t),
 661             offsetof(pid_node_t, pn_ref_link));
 662         stp->sd_wrq = NULL;
 663 
 664         return (0);
 665 }
 666 
 667 /* ARGSUSED */
 668 static void
 669 stream_head_destructor(void *buf, void *cdrarg)
 670 {
 671         stdata_t *stp = buf;
 672 
 673         mutex_destroy(&stp->sd_lock);
 674         mutex_destroy(&stp->sd_reflock);
 675         mutex_destroy(&stp->sd_qlock);
 676         mutex_destroy(&stp->sd_pid_tree_lock);
 677         cv_destroy(&stp->sd_monitor);
 678         cv_destroy(&stp->sd_iocmonitor);
 679         cv_destroy(&stp->sd_refmonitor);
 680         cv_destroy(&stp->sd_qcv);
 681         cv_destroy(&stp->sd_zcopy_wait);
 682         avl_destroy(&stp->sd_pid_tree);
 683 }
 684 
 685 /*
 686  * Constructor/destructor routines for the queue cache
 687  */
 688 /* ARGSUSED */
 689 static int
 690 queue_constructor(void *buf, void *cdrarg, int kmflags)
 691 {
 692         queinfo_t *qip = buf;
 693         queue_t *qp = &qip->qu_rqueue;
 694         queue_t *wqp = &qip->qu_wqueue;
 695         syncq_t *sq = &qip->qu_syncq;
 696 
 697         qp->q_first = NULL;
 698         qp->q_link = NULL;
 699         qp->q_count = 0;
 700         qp->q_mblkcnt = 0;
 701         qp->q_sqhead = NULL;
 702         qp->q_sqtail = NULL;
 703         qp->q_sqnext = NULL;
 704         qp->q_sqprev = NULL;
 705         qp->q_sqflags = 0;
 706         qp->q_rwcnt = 0;
 707         qp->q_spri = 0;
 708 
 709         mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
 710         cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
 711 
 712         wqp->q_first = NULL;
 713         wqp->q_link = NULL;
 714         wqp->q_count = 0;
 715         wqp->q_mblkcnt = 0;
 716         wqp->q_sqhead = NULL;
 717         wqp->q_sqtail = NULL;
 718         wqp->q_sqnext = NULL;
 719         wqp->q_sqprev = NULL;
 720         wqp->q_sqflags = 0;
 721         wqp->q_rwcnt = 0;
 722         wqp->q_spri = 0;
 723 
 724         mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
 725         cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
 726 
 727         sq->sq_head = NULL;
 728         sq->sq_tail = NULL;
 729         sq->sq_evhead = NULL;
 730         sq->sq_evtail = NULL;
 731         sq->sq_callbpend = NULL;
 732         sq->sq_outer = NULL;
 733         sq->sq_onext = NULL;
 734         sq->sq_oprev = NULL;
 735         sq->sq_next = NULL;
 736         sq->sq_svcflags = 0;
 737         sq->sq_servcount = 0;
 738         sq->sq_needexcl = 0;
 739         sq->sq_nqueues = 0;
 740         sq->sq_pri = 0;
 741 
 742         mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
 743         cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
 744         cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
 745 
 746         return (0);
 747 }
 748 
 749 /* ARGSUSED */
 750 static void
 751 queue_destructor(void *buf, void *cdrarg)
 752 {
 753         queinfo_t *qip = buf;
 754         queue_t *qp = &qip->qu_rqueue;
 755         queue_t *wqp = &qip->qu_wqueue;
 756         syncq_t *sq = &qip->qu_syncq;
 757 
 758         ASSERT(qp->q_sqhead == NULL);
 759         ASSERT(wqp->q_sqhead == NULL);
 760         ASSERT(qp->q_sqnext == NULL);
 761         ASSERT(wqp->q_sqnext == NULL);
 762         ASSERT(qp->q_rwcnt == 0);
 763         ASSERT(wqp->q_rwcnt == 0);
 764 
 765         mutex_destroy(&qp->q_lock);
 766         cv_destroy(&qp->q_wait);
 767 
 768         mutex_destroy(&wqp->q_lock);
 769         cv_destroy(&wqp->q_wait);
 770 
 771         mutex_destroy(&sq->sq_lock);
 772         cv_destroy(&sq->sq_wait);
 773         cv_destroy(&sq->sq_exitwait);
 774 }
 775 
 776 /*
 777  * Constructor/destructor routines for the syncq cache
 778  */
 779 /* ARGSUSED */
 780 static int
 781 syncq_constructor(void *buf, void *cdrarg, int kmflags)
 782 {
 783         syncq_t *sq = buf;
 784 
 785         bzero(buf, sizeof (syncq_t));
 786 
 787         mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
 788         cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
 789         cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
 790 
 791         return (0);
 792 }
 793 
 794 /* ARGSUSED */
 795 static void
 796 syncq_destructor(void *buf, void *cdrarg)
 797 {
 798         syncq_t *sq = buf;
 799 
 800         ASSERT(sq->sq_head == NULL);
 801         ASSERT(sq->sq_tail == NULL);
 802         ASSERT(sq->sq_evhead == NULL);
 803         ASSERT(sq->sq_evtail == NULL);
 804         ASSERT(sq->sq_callbpend == NULL);
 805         ASSERT(sq->sq_callbflags == 0);
 806         ASSERT(sq->sq_outer == NULL);
 807         ASSERT(sq->sq_onext == NULL);
 808         ASSERT(sq->sq_oprev == NULL);
 809         ASSERT(sq->sq_next == NULL);
 810         ASSERT(sq->sq_needexcl == 0);
 811         ASSERT(sq->sq_svcflags == 0);
 812         ASSERT(sq->sq_servcount == 0);
 813         ASSERT(sq->sq_nqueues == 0);
 814         ASSERT(sq->sq_pri == 0);
 815         ASSERT(sq->sq_count == 0);
 816         ASSERT(sq->sq_rmqcount == 0);
 817         ASSERT(sq->sq_cancelid == 0);
 818         ASSERT(sq->sq_ciputctrl == NULL);
 819         ASSERT(sq->sq_nciputctrl == 0);
 820         ASSERT(sq->sq_type == 0);
 821         ASSERT(sq->sq_flags == 0);
 822 
 823         mutex_destroy(&sq->sq_lock);
 824         cv_destroy(&sq->sq_wait);
 825         cv_destroy(&sq->sq_exitwait);
 826 }
 827 
 828 /* ARGSUSED */
 829 static int
 830 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
 831 {
 832         ciputctrl_t *cip = buf;
 833         int i;
 834 
 835         for (i = 0; i < n_ciputctrl; i++) {
 836                 cip[i].ciputctrl_count = SQ_FASTPUT;
 837                 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
 838         }
 839 
 840         return (0);
 841 }
 842 
 843 /* ARGSUSED */
 844 static void
 845 ciputctrl_destructor(void *buf, void *cdrarg)
 846 {
 847         ciputctrl_t *cip = buf;
 848         int i;
 849 
 850         for (i = 0; i < n_ciputctrl; i++) {
 851                 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
 852                 mutex_destroy(&cip[i].ciputctrl_lock);
 853         }
 854 }
 855 
 856 /*
 857  * Init routine run from main at boot time.
 858  */
 859 void
 860 strinit(void)
 861 {
 862         int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
 863 
 864         stream_head_cache = kmem_cache_create("stream_head_cache",
 865             sizeof (stdata_t), 0,
 866             stream_head_constructor, stream_head_destructor, NULL,
 867             NULL, NULL, 0);
 868 
 869         queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
 870             queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
 871 
 872         syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
 873             syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
 874 
 875         qband_cache = kmem_cache_create("qband_cache",
 876             sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 877 
 878         linkinfo_cache = kmem_cache_create("linkinfo_cache",
 879             sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 880 
 881         n_ciputctrl = ncpus;
 882         n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
 883         ASSERT(n_ciputctrl >= 1);
 884         n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
 885         if (n_ciputctrl >= min_n_ciputctrl) {
 886                 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
 887                     sizeof (ciputctrl_t) * n_ciputctrl,
 888                     sizeof (ciputctrl_t), ciputctrl_constructor,
 889                     ciputctrl_destructor, NULL, NULL, NULL, 0);
 890         }
 891 
 892         streams_taskq = system_taskq;
 893 
 894         if (streams_taskq == NULL)
 895                 panic("strinit: no memory for streams taskq!");
 896 
 897         bc_bkgrnd_thread = thread_create(NULL, 0,
 898             streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 899 
 900         streams_qbkgrnd_thread = thread_create(NULL, 0,
 901             streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 902 
 903         streams_sqbkgrnd_thread = thread_create(NULL, 0,
 904             streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
 905 
 906         /*
 907          * Create STREAMS kstats.
 908          */
 909         str_kstat = kstat_create("streams", 0, "strstat",
 910             "net", KSTAT_TYPE_NAMED,
 911             sizeof (str_statistics) / sizeof (kstat_named_t),
 912             KSTAT_FLAG_VIRTUAL);
 913 
 914         if (str_kstat != NULL) {
 915                 str_kstat->ks_data = &str_statistics;
 916                 kstat_install(str_kstat);
 917         }
 918 
 919         /*
 920          * TPI support routine initialisation.
 921          */
 922         tpi_init();
 923 
 924         /*
 925          * Handle to have autopush and persistent link information per
 926          * zone.
 927          * Note: uses shutdown hook instead of destroy hook so that the
 928          * persistent links can be torn down before the destroy hooks
 929          * in the TCP/IP stack are called.
 930          */
 931         netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
 932             str_stack_fini);
 933 }
 934 
 935 void
 936 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
 937 {
 938         struct stdata *stp;
 939 
 940         ASSERT(vp->v_stream);
 941         stp = vp->v_stream;
 942         /* Have to hold sd_lock to prevent siglist from changing */
 943         mutex_enter(&stp->sd_lock);
 944         if (stp->sd_sigflags & event)
 945                 strsendsig(stp->sd_siglist, event, band, error);
 946         mutex_exit(&stp->sd_lock);
 947 }
 948 
 949 /*
 950  * Send the "sevent" set of signals to a process.
 951  * This might send more than one signal if the process is registered
 952  * for multiple events. The caller should pass in an sevent that only
 953  * includes the events for which the process has registered.
 954  */
 955 static void
 956 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
 957     uchar_t band, int error)
 958 {
 959         ASSERT(MUTEX_HELD(&proc->p_lock));
 960 
 961         info->si_band = 0;
 962         info->si_errno = 0;
 963 
 964         if (sevent & S_ERROR) {
 965                 sevent &= ~S_ERROR;
 966                 info->si_code = POLL_ERR;
 967                 info->si_errno = error;
 968                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 969                     "strsendsig:proc %p info %p", proc, info);
 970                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 971                 info->si_errno = 0;
 972         }
 973         if (sevent & S_HANGUP) {
 974                 sevent &= ~S_HANGUP;
 975                 info->si_code = POLL_HUP;
 976                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 977                     "strsendsig:proc %p info %p", proc, info);
 978                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 979         }
 980         if (sevent & S_HIPRI) {
 981                 sevent &= ~S_HIPRI;
 982                 info->si_code = POLL_PRI;
 983                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
 984                     "strsendsig:proc %p info %p", proc, info);
 985                 sigaddq(proc, NULL, info, KM_NOSLEEP);
 986         }
 987         if (sevent & S_RDBAND) {
 988                 sevent &= ~S_RDBAND;
 989                 if (events & S_BANDURG)
 990                         sigtoproc(proc, NULL, SIGURG);
 991                 else
 992                         sigtoproc(proc, NULL, SIGPOLL);
 993         }
 994         if (sevent & S_WRBAND) {
 995                 sevent &= ~S_WRBAND;
 996                 sigtoproc(proc, NULL, SIGPOLL);
 997         }
 998         if (sevent & S_INPUT) {
 999                 sevent &= ~S_INPUT;
1000                 info->si_code = POLL_IN;
1001                 info->si_band = band;
1002                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1003                     "strsendsig:proc %p info %p", proc, info);
1004                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1005                 info->si_band = 0;
1006         }
1007         if (sevent & S_OUTPUT) {
1008                 sevent &= ~S_OUTPUT;
1009                 info->si_code = POLL_OUT;
1010                 info->si_band = band;
1011                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1012                     "strsendsig:proc %p info %p", proc, info);
1013                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1014                 info->si_band = 0;
1015         }
1016         if (sevent & S_MSG) {
1017                 sevent &= ~S_MSG;
1018                 info->si_code = POLL_MSG;
1019                 info->si_band = band;
1020                 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1021                     "strsendsig:proc %p info %p", proc, info);
1022                 sigaddq(proc, NULL, info, KM_NOSLEEP);
1023                 info->si_band = 0;
1024         }
1025         if (sevent & S_RDNORM) {
1026                 sevent &= ~S_RDNORM;
1027                 sigtoproc(proc, NULL, SIGPOLL);
1028         }
1029         if (sevent != 0) {
1030                 panic("strsendsig: unknown event(s) %x", sevent);
1031         }
1032 }
1033 
1034 /*
1035  * Send SIGPOLL/SIGURG signal to all processes and process groups
1036  * registered on the given signal list that want a signal for at
1037  * least one of the specified events.
1038  *
1039  * Must be called with exclusive access to siglist (caller holding sd_lock).
1040  *
1041  * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1042  * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1043  * while it is in the siglist.
1044  *
1045  * For performance reasons (MP scalability) the code drops pidlock
1046  * when sending signals to a single process.
1047  * When sending to a process group the code holds
1048  * pidlock to prevent the membership in the process group from changing
1049  * while walking the p_pglink list.
1050  */
1051 void
1052 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1053 {
1054         strsig_t *ssp;
1055         k_siginfo_t info;
1056         struct pid *pidp;
1057         proc_t  *proc;
1058 
1059         info.si_signo = SIGPOLL;
1060         info.si_errno = 0;
1061         for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1062                 int sevent;
1063 
1064                 sevent = ssp->ss_events & event;
1065                 if (sevent == 0)
1066                         continue;
1067 
1068                 if ((pidp = ssp->ss_pidp) == NULL) {
1069                         /* pid was released but still on event list */
1070                         continue;
1071                 }
1072 
1073 
1074                 if (ssp->ss_pid > 0) {
1075                         /*
1076                          * XXX This unfortunately still generates
1077                          * a signal when a fd is closed but
1078                          * the proc is active.
1079                          */
1080                         ASSERT(ssp->ss_pid == pidp->pid_id);
1081 
1082                         mutex_enter(&pidlock);
1083                         proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1084                         if (proc == NULL) {
1085                                 mutex_exit(&pidlock);
1086                                 continue;
1087                         }
1088                         mutex_enter(&proc->p_lock);
1089                         mutex_exit(&pidlock);
1090                         dosendsig(proc, ssp->ss_events, sevent, &info,
1091                             band, error);
1092                         mutex_exit(&proc->p_lock);
1093                 } else {
1094                         /*
1095                          * Send to process group. Hold pidlock across
1096                          * calls to dosendsig().
1097                          */
1098                         pid_t pgrp = -ssp->ss_pid;
1099 
1100                         mutex_enter(&pidlock);
1101                         proc = pgfind_zone(pgrp, ALL_ZONES);
1102                         while (proc != NULL) {
1103                                 mutex_enter(&proc->p_lock);
1104                                 dosendsig(proc, ssp->ss_events, sevent,
1105                                     &info, band, error);
1106                                 mutex_exit(&proc->p_lock);
1107                                 proc = proc->p_pglink;
1108                         }
1109                         mutex_exit(&pidlock);
1110                 }
1111         }
1112 }
1113 
1114 /*
1115  * Attach a stream device or module.
1116  * qp is a read queue; the new queue goes in so its next
1117  * read ptr is the argument, and the write queue corresponding
1118  * to the argument points to this queue. Return 0 on success,
1119  * or a non-zero errno on failure.
1120  */
1121 int
1122 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1123     boolean_t is_insert)
1124 {
1125         major_t                 major;
1126         cdevsw_impl_t           *dp;
1127         struct streamtab        *str;
1128         queue_t                 *rq;
1129         queue_t                 *wrq;
1130         uint32_t                qflag;
1131         uint32_t                sqtype;
1132         perdm_t                 *dmp;
1133         int                     error;
1134         int                     sflag;
1135 
1136         rq = allocq();
1137         wrq = _WR(rq);
1138         STREAM(rq) = STREAM(wrq) = STREAM(qp);
1139 
1140         if (fp != NULL) {
1141                 str = fp->f_str;
1142                 qflag = fp->f_qflag;
1143                 sqtype = fp->f_sqtype;
1144                 dmp = fp->f_dmp;
1145                 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1146                 sflag = MODOPEN;
1147 
1148                 /*
1149                  * stash away a pointer to the module structure so we can
1150                  * unref it in qdetach.
1151                  */
1152                 rq->q_fp = fp;
1153         } else {
1154                 ASSERT(!is_insert);
1155 
1156                 major = getmajor(*devp);
1157                 dp = &devimpl[major];
1158 
1159                 str = dp->d_str;
1160                 ASSERT(str == STREAMSTAB(major));
1161 
1162                 qflag = dp->d_qflag;
1163                 ASSERT(qflag & QISDRV);
1164                 sqtype = dp->d_sqtype;
1165 
1166                 /* create perdm_t if needed */
1167                 if (NEED_DM(dp->d_dmp, qflag))
1168                         dp->d_dmp = hold_dm(str, qflag, sqtype);
1169 
1170                 dmp = dp->d_dmp;
1171                 sflag = 0;
1172         }
1173 
1174         TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1175             "qattach:qflag == %X(%X)", qflag, *devp);
1176 
1177         /* setq might sleep in allocator - avoid holding locks. */
1178         setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1179 
1180         /*
1181          * Before calling the module's open routine, set up the q_next
1182          * pointer for inserting a module in the middle of a stream.
1183          *
1184          * Note that we can always set _QINSERTING and set up q_next
1185          * pointer for both inserting and pushing a module.  Then there
1186          * is no need for the is_insert parameter.  In insertq(), called
1187          * by qprocson(), assume that q_next of the new module always points
1188          * to the correct queue and use it for insertion.  Everything should
1189          * work out fine.  But in the first release of _I_INSERT, we
1190          * distinguish between inserting and pushing to make sure that
1191          * pushing a module follows the same code path as before.
1192          */
1193         if (is_insert) {
1194                 rq->q_flag |= _QINSERTING;
1195                 rq->q_next = qp;
1196         }
1197 
1198         /*
1199          * If there is an outer perimeter get exclusive access during
1200          * the open procedure.  Bump up the reference count on the queue.
1201          */
1202         entersq(rq->q_syncq, SQ_OPENCLOSE);
1203         error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1204         if (error != 0)
1205                 goto failed;
1206         leavesq(rq->q_syncq, SQ_OPENCLOSE);
1207         ASSERT(qprocsareon(rq));
1208         return (0);
1209 
1210 failed:
1211         rq->q_flag &= ~_QINSERTING;
1212         if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1213                 qprocsoff(rq);
1214         leavesq(rq->q_syncq, SQ_OPENCLOSE);
1215         rq->q_next = wrq->q_next = NULL;
1216         qdetach(rq, 0, 0, crp, B_FALSE);
1217         return (error);
1218 }
1219 
1220 /*
1221  * Handle second open of stream. For modules, set the
1222  * last argument to MODOPEN and do not pass any open flags.
1223  * Ignore dummydev since this is not the first open.
1224  */
1225 int
1226 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1227 {
1228         int     error;
1229         dev_t dummydev;
1230         queue_t *wqp = _WR(qp);
1231 
1232         ASSERT(qp->q_flag & QREADR);
1233         entersq(qp->q_syncq, SQ_OPENCLOSE);
1234 
1235         dummydev = *devp;
1236         if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1237             (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1238                 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1239                 mutex_enter(&STREAM(qp)->sd_lock);
1240                 qp->q_stream->sd_flag |= STREOPENFAIL;
1241                 mutex_exit(&STREAM(qp)->sd_lock);
1242                 return (error);
1243         }
1244         leavesq(qp->q_syncq, SQ_OPENCLOSE);
1245 
1246         /*
1247          * successful open should have done qprocson()
1248          */
1249         ASSERT(qprocsareon(_RD(qp)));
1250         return (0);
1251 }
1252 
1253 /*
1254  * Detach a stream module or device.
1255  * If clmode == 1 then the module or driver was opened and its
1256  * close routine must be called. If clmode == 0, the module
1257  * or driver was never opened or the open failed, and so its close
1258  * should not be called.
1259  */
1260 void
1261 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1262 {
1263         queue_t *wqp = _WR(qp);
1264         ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1265 
1266         if (STREAM_NEEDSERVICE(STREAM(qp)))
1267                 stream_runservice(STREAM(qp));
1268 
1269         if (clmode) {
1270                 /*
1271                  * Make sure that all the messages on the write side syncq are
1272                  * processed and nothing is left. Since we are closing, no new
1273                  * messages may appear there.
1274                  */
1275                 wait_q_syncq(wqp);
1276 
1277                 entersq(qp->q_syncq, SQ_OPENCLOSE);
1278                 if (is_remove) {
1279                         mutex_enter(QLOCK(qp));
1280                         qp->q_flag |= _QREMOVING;
1281                         mutex_exit(QLOCK(qp));
1282                 }
1283                 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1284                 /*
1285                  * Check that qprocsoff() was actually called.
1286                  */
1287                 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1288 
1289                 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1290         } else {
1291                 disable_svc(qp);
1292         }
1293 
1294         /*
1295          * Allow any threads blocked in entersq to proceed and discover
1296          * the QWCLOSE is set.
1297          * Note: This assumes that all users of entersq check QWCLOSE.
1298          * Currently runservice is the only entersq that can happen
1299          * after removeq has finished.
1300          * Removeq will have discarded all messages destined to the closing
1301          * pair of queues from the syncq.
1302          * NOTE: Calling a function inside an assert is unconventional.
1303          * However, it does not cause any problem since flush_syncq() does
1304          * not change any state except when it returns non-zero i.e.
1305          * when the assert will trigger.
1306          */
1307         ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1308         ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1309         ASSERT((qp->q_flag & QPERMOD) ||
1310             ((qp->q_syncq->sq_head == NULL) &&
1311             (wqp->q_syncq->sq_head == NULL)));
1312 
1313         /* release any fmodsw_impl_t structure held on behalf of the queue */
1314         ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1315         if (qp->q_fp != NULL)
1316                 fmodsw_rele(qp->q_fp);
1317 
1318         /* freeq removes us from the outer perimeter if any */
1319         freeq(qp);
1320 }
1321 
1322 /* Prevent service procedures from being called */
1323 void
1324 disable_svc(queue_t *qp)
1325 {
1326         queue_t *wqp = _WR(qp);
1327 
1328         ASSERT(qp->q_flag & QREADR);
1329         mutex_enter(QLOCK(qp));
1330         qp->q_flag |= QWCLOSE;
1331         mutex_exit(QLOCK(qp));
1332         mutex_enter(QLOCK(wqp));
1333         wqp->q_flag |= QWCLOSE;
1334         mutex_exit(QLOCK(wqp));
1335 }
1336 
1337 /* Allow service procedures to be called again */
1338 void
1339 enable_svc(queue_t *qp)
1340 {
1341         queue_t *wqp = _WR(qp);
1342 
1343         ASSERT(qp->q_flag & QREADR);
1344         mutex_enter(QLOCK(qp));
1345         qp->q_flag &= ~QWCLOSE;
1346         mutex_exit(QLOCK(qp));
1347         mutex_enter(QLOCK(wqp));
1348         wqp->q_flag &= ~QWCLOSE;
1349         mutex_exit(QLOCK(wqp));
1350 }
1351 
1352 /*
1353  * Remove queue from qhead/qtail if it is enabled.
1354  * Only reset QENAB if the queue was removed from the runlist.
1355  * A queue goes through 3 stages:
1356  *      It is on the service list and QENAB is set.
1357  *      It is removed from the service list but QENAB is still set.
1358  *      QENAB gets changed to QINSERVICE.
1359  *      QINSERVICE is reset (when the service procedure is done)
1360  * Thus we can not reset QENAB unless we actually removed it from the service
1361  * queue.
1362  */
1363 void
1364 remove_runlist(queue_t *qp)
1365 {
1366         if (qp->q_flag & QENAB && qhead != NULL) {
1367                 queue_t *q_chase;
1368                 queue_t *q_curr;
1369                 int removed;
1370 
1371                 mutex_enter(&service_queue);
1372                 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1373                 mutex_exit(&service_queue);
1374                 if (removed) {
1375                         STRSTAT(qremoved);
1376                         qp->q_flag &= ~QENAB;
1377                 }
1378         }
1379 }
1380 
1381 
1382 /*
1383  * Wait for any pending service processing to complete.
1384  * The removal of queues from the runlist is not atomic with the
1385  * clearing of the QENABLED flag and setting the INSERVICE flag.
1386  * consequently it is possible for remove_runlist in strclose
1387  * to not find the queue on the runlist but for it to be QENABLED
1388  * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1389  * as well as INSERVICE.
1390  */
1391 void
1392 wait_svc(queue_t *qp)
1393 {
1394         queue_t *wqp = _WR(qp);
1395 
1396         ASSERT(qp->q_flag & QREADR);
1397 
1398         /*
1399          * Try to remove queues from qhead/qtail list.
1400          */
1401         if (qhead != NULL) {
1402                 remove_runlist(qp);
1403                 remove_runlist(wqp);
1404         }
1405         /*
1406          * Wait till the syncqs associated with the queue disappear from the
1407          * background processing list.
1408          * This only needs to be done for non-PERMOD perimeters since
1409          * for PERMOD perimeters the syncq may be shared and will only be freed
1410          * when the last module/driver is unloaded.
1411          * If for PERMOD perimeters queue was on the syncq list, removeq()
1412          * should call propagate_syncq() or drain_syncq() for it. Both of these
1413          * functions remove the queue from its syncq list, so sqthread will not
1414          * try to access the queue.
1415          */
1416         if (!(qp->q_flag & QPERMOD)) {
1417                 syncq_t *rsq = qp->q_syncq;
1418                 syncq_t *wsq = wqp->q_syncq;
1419 
1420                 /*
1421                  * Disable rsq and wsq and wait for any background processing of
1422                  * syncq to complete.
1423                  */
1424                 wait_sq_svc(rsq);
1425                 if (wsq != rsq)
1426                         wait_sq_svc(wsq);
1427         }
1428 
1429         mutex_enter(QLOCK(qp));
1430         while (qp->q_flag & (QINSERVICE|QENAB))
1431                 cv_wait(&qp->q_wait, QLOCK(qp));
1432         mutex_exit(QLOCK(qp));
1433         mutex_enter(QLOCK(wqp));
1434         while (wqp->q_flag & (QINSERVICE|QENAB))
1435                 cv_wait(&wqp->q_wait, QLOCK(wqp));
1436         mutex_exit(QLOCK(wqp));
1437 }
1438 
1439 /*
1440  * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1441  * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1442  * also be set, and is passed through to allocb_cred_wait().
1443  *
1444  * Returns errno on failure, zero on success.
1445  */
1446 int
1447 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1448 {
1449         mblk_t *tmp;
1450         ssize_t  count;
1451         int error = 0;
1452 
1453         ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1454             (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1455 
1456         if (bp->b_datap->db_type == M_IOCTL) {
1457                 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1458         } else {
1459                 ASSERT(bp->b_datap->db_type == M_COPYIN);
1460                 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1461         }
1462         /*
1463          * strdoioctl validates ioc_count, so if this assert fails it
1464          * cannot be due to user error.
1465          */
1466         ASSERT(count >= 0);
1467 
1468         if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1469             curproc->p_pid)) == NULL) {
1470                 return (error);
1471         }
1472         error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1473         if (error != 0) {
1474                 freeb(tmp);
1475                 return (error);
1476         }
1477         DB_CPID(tmp) = curproc->p_pid;
1478         tmp->b_wptr += count;
1479         bp->b_cont = tmp;
1480 
1481         return (0);
1482 }
1483 
1484 /*
1485  * Copy ioctl data to user-land. Return non-zero errno on failure,
1486  * 0 for success.
1487  */
1488 int
1489 getiocd(mblk_t *bp, char *arg, int copymode)
1490 {
1491         ssize_t count;
1492         size_t  n;
1493         int     error;
1494 
1495         if (bp->b_datap->db_type == M_IOCACK)
1496                 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1497         else {
1498                 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1499                 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1500         }
1501         ASSERT(count >= 0);
1502 
1503         for (bp = bp->b_cont; bp && count;
1504             count -= n, bp = bp->b_cont, arg += n) {
1505                 n = MIN(count, bp->b_wptr - bp->b_rptr);
1506                 error = strcopyout(bp->b_rptr, arg, n, copymode);
1507                 if (error)
1508                         return (error);
1509         }
1510         ASSERT(count == 0);
1511         return (0);
1512 }
1513 
1514 /*
1515  * Allocate a linkinfo entry given the write queue of the
1516  * bottom module of the top stream and the write queue of the
1517  * stream head of the bottom stream.
1518  */
1519 linkinfo_t *
1520 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1521 {
1522         linkinfo_t *linkp;
1523 
1524         linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1525 
1526         linkp->li_lblk.l_qtop = qup;
1527         linkp->li_lblk.l_qbot = qdown;
1528         linkp->li_fpdown = fpdown;
1529 
1530         mutex_enter(&strresources);
1531         linkp->li_next = linkinfo_list;
1532         linkp->li_prev = NULL;
1533         if (linkp->li_next)
1534                 linkp->li_next->li_prev = linkp;
1535         linkinfo_list = linkp;
1536         linkp->li_lblk.l_index = ++lnk_id;
1537         ASSERT(lnk_id != 0);    /* this should never wrap in practice */
1538         mutex_exit(&strresources);
1539 
1540         return (linkp);
1541 }
1542 
1543 /*
1544  * Free a linkinfo entry.
1545  */
1546 void
1547 lbfree(linkinfo_t *linkp)
1548 {
1549         mutex_enter(&strresources);
1550         if (linkp->li_next)
1551                 linkp->li_next->li_prev = linkp->li_prev;
1552         if (linkp->li_prev)
1553                 linkp->li_prev->li_next = linkp->li_next;
1554         else
1555                 linkinfo_list = linkp->li_next;
1556         mutex_exit(&strresources);
1557 
1558         kmem_cache_free(linkinfo_cache, linkp);
1559 }
1560 
1561 /*
1562  * Check for a potential linking cycle.
1563  * Return 1 if a link will result in a cycle,
1564  * and 0 otherwise.
1565  */
1566 int
1567 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1568 {
1569         struct mux_node *np;
1570         struct mux_edge *ep;
1571         int i;
1572         major_t lomaj;
1573         major_t upmaj;
1574         /*
1575          * if the lower stream is a pipe/FIFO, return, since link
1576          * cycles can not happen on pipes/FIFOs
1577          */
1578         if (lostp->sd_vnode->v_type == VFIFO)
1579                 return (0);
1580 
1581         for (i = 0; i < ss->ss_devcnt; i++) {
1582                 np = &ss->ss_mux_nodes[i];
1583                 MUX_CLEAR(np);
1584         }
1585         lomaj = getmajor(lostp->sd_vnode->v_rdev);
1586         upmaj = getmajor(upstp->sd_vnode->v_rdev);
1587         np = &ss->ss_mux_nodes[lomaj];
1588         for (;;) {
1589                 if (!MUX_DIDVISIT(np)) {
1590                         if (np->mn_imaj == upmaj)
1591                                 return (1);
1592                         if (np->mn_outp == NULL) {
1593                                 MUX_VISIT(np);
1594                                 if (np->mn_originp == NULL)
1595                                         return (0);
1596                                 np = np->mn_originp;
1597                                 continue;
1598                         }
1599                         MUX_VISIT(np);
1600                         np->mn_startp = np->mn_outp;
1601                 } else {
1602                         if (np->mn_startp == NULL) {
1603                                 if (np->mn_originp == NULL)
1604                                         return (0);
1605                                 else {
1606                                         np = np->mn_originp;
1607                                         continue;
1608                                 }
1609                         }
1610                         /*
1611                          * If ep->me_nodep is a FIFO (me_nodep == NULL),
1612                          * ignore the edge and move on. ep->me_nodep gets
1613                          * set to NULL in mux_addedge() if it is a FIFO.
1614                          *
1615                          */
1616                         ep = np->mn_startp;
1617                         np->mn_startp = ep->me_nextp;
1618                         if (ep->me_nodep == NULL)
1619                                 continue;
1620                         ep->me_nodep->mn_originp = np;
1621                         np = ep->me_nodep;
1622                 }
1623         }
1624 }
1625 
1626 /*
1627  * Find linkinfo entry corresponding to the parameters.
1628  */
1629 linkinfo_t *
1630 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1631 {
1632         linkinfo_t *linkp;
1633         struct mux_edge *mep;
1634         struct mux_node *mnp;
1635         queue_t *qup;
1636 
1637         mutex_enter(&strresources);
1638         if ((type & LINKTYPEMASK) == LINKNORMAL) {
1639                 qup = getendq(stp->sd_wrq);
1640                 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1641                         if ((qup == linkp->li_lblk.l_qtop) &&
1642                             (!index || (index == linkp->li_lblk.l_index))) {
1643                                 mutex_exit(&strresources);
1644                                 return (linkp);
1645                         }
1646                 }
1647         } else {
1648                 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1649                 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1650                 mep = mnp->mn_outp;
1651                 while (mep) {
1652                         if ((index == 0) || (index == mep->me_muxid))
1653                                 break;
1654                         mep = mep->me_nextp;
1655                 }
1656                 if (!mep) {
1657                         mutex_exit(&strresources);
1658                         return (NULL);
1659                 }
1660                 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1661                         if ((!linkp->li_lblk.l_qtop) &&
1662                             (mep->me_muxid == linkp->li_lblk.l_index)) {
1663                                 mutex_exit(&strresources);
1664                                 return (linkp);
1665                         }
1666                 }
1667         }
1668         mutex_exit(&strresources);
1669         return (NULL);
1670 }
1671 
1672 /*
1673  * Given a queue ptr, follow the chain of q_next pointers until you reach the
1674  * last queue on the chain and return it.
1675  */
1676 queue_t *
1677 getendq(queue_t *q)
1678 {
1679         ASSERT(q != NULL);
1680         while (_SAMESTR(q))
1681                 q = q->q_next;
1682         return (q);
1683 }
1684 
1685 /*
1686  * Wait for the syncq count to drop to zero.
1687  * sq could be either outer or inner.
1688  */
1689 
1690 static void
1691 wait_syncq(syncq_t *sq)
1692 {
1693         uint16_t count;
1694 
1695         mutex_enter(SQLOCK(sq));
1696         count = sq->sq_count;
1697         SQ_PUTLOCKS_ENTER(sq);
1698         SUM_SQ_PUTCOUNTS(sq, count);
1699         while (count != 0) {
1700                 sq->sq_flags |= SQ_WANTWAKEUP;
1701                 SQ_PUTLOCKS_EXIT(sq);
1702                 cv_wait(&sq->sq_wait, SQLOCK(sq));
1703                 count = sq->sq_count;
1704                 SQ_PUTLOCKS_ENTER(sq);
1705                 SUM_SQ_PUTCOUNTS(sq, count);
1706         }
1707         SQ_PUTLOCKS_EXIT(sq);
1708         mutex_exit(SQLOCK(sq));
1709 }
1710 
1711 /*
1712  * Wait while there are any messages for the queue in its syncq.
1713  */
1714 static void
1715 wait_q_syncq(queue_t *q)
1716 {
1717         if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1718                 syncq_t *sq = q->q_syncq;
1719 
1720                 mutex_enter(SQLOCK(sq));
1721                 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1722                         sq->sq_flags |= SQ_WANTWAKEUP;
1723                         cv_wait(&sq->sq_wait, SQLOCK(sq));
1724                 }
1725                 mutex_exit(SQLOCK(sq));
1726         }
1727 }
1728 
1729 
1730 int
1731 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1732     int lhlink)
1733 {
1734         struct stdata *stp;
1735         struct strioctl strioc;
1736         struct linkinfo *linkp;
1737         struct stdata *stpdown;
1738         struct streamtab *str;
1739         queue_t *passq;
1740         syncq_t *passyncq;
1741         queue_t *rq;
1742         cdevsw_impl_t *dp;
1743         uint32_t qflag;
1744         uint32_t sqtype;
1745         perdm_t *dmp;
1746         int error = 0;
1747         netstack_t *ns;
1748         str_stack_t *ss;
1749 
1750         stp = vp->v_stream;
1751         TRACE_1(TR_FAC_STREAMS_FR,
1752             TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1753         /*
1754          * Test for invalid upper stream
1755          */
1756         if (stp->sd_flag & STRHUP) {
1757                 return (ENXIO);
1758         }
1759         if (vp->v_type == VFIFO) {
1760                 return (EINVAL);
1761         }
1762         if (stp->sd_strtab == NULL) {
1763                 return (EINVAL);
1764         }
1765         if (!stp->sd_strtab->st_muxwinit) {
1766                 return (EINVAL);
1767         }
1768         if (fpdown == NULL) {
1769                 return (EBADF);
1770         }
1771         ns = netstack_find_by_cred(crp);
1772         ASSERT(ns != NULL);
1773         ss = ns->netstack_str;
1774         ASSERT(ss != NULL);
1775 
1776         if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1777                 netstack_rele(ss->ss_netstack);
1778                 return (EINVAL);
1779         }
1780         mutex_enter(&muxifier);
1781         if (stp->sd_flag & STPLEX) {
1782                 mutex_exit(&muxifier);
1783                 netstack_rele(ss->ss_netstack);
1784                 return (ENXIO);
1785         }
1786 
1787         /*
1788          * Test for invalid lower stream.
1789          * The check for the v_type != VFIFO and having a major
1790          * number not >= devcnt is done to avoid problems with
1791          * adding mux_node entry past the end of mux_nodes[].
1792          * For FIFO's we don't add an entry so this isn't a
1793          * problem.
1794          */
1795         if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1796             (stpdown == stp) || (stpdown->sd_flag &
1797             (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1798             ((stpdown->sd_vnode->v_type != VFIFO) &&
1799             (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1800             linkcycle(stp, stpdown, ss)) {
1801                 mutex_exit(&muxifier);
1802                 netstack_rele(ss->ss_netstack);
1803                 return (EINVAL);
1804         }
1805         TRACE_1(TR_FAC_STREAMS_FR,
1806             TR_STPDOWN, "stpdown:%p", stpdown);
1807         rq = getendq(stp->sd_wrq);
1808         if (cmd == I_PLINK)
1809                 rq = NULL;
1810 
1811         linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1812 
1813         strioc.ic_cmd = cmd;
1814         strioc.ic_timout = INFTIM;
1815         strioc.ic_len = sizeof (struct linkblk);
1816         strioc.ic_dp = (char *)&linkp->li_lblk;
1817 
1818         /*
1819          * STRPLUMB protects plumbing changes and should be set before
1820          * link_addpassthru()/link_rempassthru() are called, so it is set here
1821          * and cleared in the end of mlink when passthru queue is removed.
1822          * Setting of STRPLUMB prevents reopens of the stream while passthru
1823          * queue is in-place (it is not a proper module and doesn't have open
1824          * entry point).
1825          *
1826          * STPLEX prevents any threads from entering the stream from above. It
1827          * can't be set before the call to link_addpassthru() because putnext
1828          * from below may cause stream head I/O routines to be called and these
1829          * routines assert that STPLEX is not set. After link_addpassthru()
1830          * nothing may come from below since the pass queue syncq is blocked.
1831          * Note also that STPLEX should be cleared before the call to
1832          * link_rempassthru() since when messages start flowing to the stream
1833          * head (e.g. because of message propagation from the pass queue) stream
1834          * head I/O routines may be called with STPLEX flag set.
1835          *
1836          * When STPLEX is set, nothing may come into the stream from above and
1837          * it is safe to do a setq which will change stream head. So, the
1838          * correct sequence of actions is:
1839          *
1840          * 1) Set STRPLUMB
1841          * 2) Call link_addpassthru()
1842          * 3) Set STPLEX
1843          * 4) Call setq and update the stream state
1844          * 5) Clear STPLEX
1845          * 6) Call link_rempassthru()
1846          * 7) Clear STRPLUMB
1847          *
1848          * The same sequence applies to munlink() code.
1849          */
1850         mutex_enter(&stpdown->sd_lock);
1851         stpdown->sd_flag |= STRPLUMB;
1852         mutex_exit(&stpdown->sd_lock);
1853         /*
1854          * Add passthru queue below lower mux. This will block
1855          * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1856          */
1857         passq = link_addpassthru(stpdown);
1858 
1859         mutex_enter(&stpdown->sd_lock);
1860         stpdown->sd_flag |= STPLEX;
1861         mutex_exit(&stpdown->sd_lock);
1862 
1863         rq = _RD(stpdown->sd_wrq);
1864         /*
1865          * There may be messages in the streamhead's syncq due to messages
1866          * that arrived before link_addpassthru() was done. To avoid
1867          * background processing of the syncq happening simultaneous with
1868          * setq processing, we disable the streamhead syncq and wait until
1869          * existing background thread finishes working on it.
1870          */
1871         wait_sq_svc(rq->q_syncq);
1872         passyncq = passq->q_syncq;
1873         if (!(passyncq->sq_flags & SQ_BLOCKED))
1874                 blocksq(passyncq, SQ_BLOCKED, 0);
1875 
1876         ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1877         ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1878         rq->q_ptr = _WR(rq)->q_ptr = NULL;
1879 
1880         /* setq might sleep in allocator - avoid holding locks. */
1881         /* Note: we are holding muxifier here. */
1882 
1883         str = stp->sd_strtab;
1884         dp = &devimpl[getmajor(vp->v_rdev)];
1885         ASSERT(dp->d_str == str);
1886 
1887         qflag = dp->d_qflag;
1888         sqtype = dp->d_sqtype;
1889 
1890         /* create perdm_t if needed */
1891         if (NEED_DM(dp->d_dmp, qflag))
1892                 dp->d_dmp = hold_dm(str, qflag, sqtype);
1893 
1894         dmp = dp->d_dmp;
1895 
1896         setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1897             B_TRUE);
1898 
1899         /*
1900          * XXX Remove any "odd" messages from the queue.
1901          * Keep only M_DATA, M_PROTO, M_PCPROTO.
1902          */
1903         error = strdoioctl(stp, &strioc, FNATIVE,
1904             K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1905         if (error != 0) {
1906                 lbfree(linkp);
1907 
1908                 if (!(passyncq->sq_flags & SQ_BLOCKED))
1909                         blocksq(passyncq, SQ_BLOCKED, 0);
1910                 /*
1911                  * Restore the stream head queue and then remove
1912                  * the passq. Turn off STPLEX before we turn on
1913                  * the stream by removing the passq.
1914                  */
1915                 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1916                 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1917                     B_TRUE);
1918 
1919                 mutex_enter(&stpdown->sd_lock);
1920                 stpdown->sd_flag &= ~STPLEX;
1921                 mutex_exit(&stpdown->sd_lock);
1922 
1923                 link_rempassthru(passq);
1924 
1925                 mutex_enter(&stpdown->sd_lock);
1926                 stpdown->sd_flag &= ~STRPLUMB;
1927                 /* Wakeup anyone waiting for STRPLUMB to clear. */
1928                 cv_broadcast(&stpdown->sd_monitor);
1929                 mutex_exit(&stpdown->sd_lock);
1930 
1931                 mutex_exit(&muxifier);
1932                 netstack_rele(ss->ss_netstack);
1933                 return (error);
1934         }
1935         mutex_enter(&fpdown->f_tlock);
1936         fpdown->f_count++;
1937         mutex_exit(&fpdown->f_tlock);
1938 
1939         /*
1940          * if we've made it here the linkage is all set up so we should also
1941          * set up the layered driver linkages
1942          */
1943 
1944         ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1945         if (cmd == I_LINK) {
1946                 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1947         } else {
1948                 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1949         }
1950 
1951         link_rempassthru(passq);
1952 
1953         mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1954 
1955         /*
1956          * Mark the upper stream as having dependent links
1957          * so that strclose can clean it up.
1958          */
1959         if (cmd == I_LINK) {
1960                 mutex_enter(&stp->sd_lock);
1961                 stp->sd_flag |= STRHASLINKS;
1962                 mutex_exit(&stp->sd_lock);
1963         }
1964         /*
1965          * Wake up any other processes that may have been
1966          * waiting on the lower stream. These will all
1967          * error out.
1968          */
1969         mutex_enter(&stpdown->sd_lock);
1970         /* The passthru module is removed so we may release STRPLUMB */
1971         stpdown->sd_flag &= ~STRPLUMB;
1972         cv_broadcast(&rq->q_wait);
1973         cv_broadcast(&_WR(rq)->q_wait);
1974         cv_broadcast(&stpdown->sd_monitor);
1975         mutex_exit(&stpdown->sd_lock);
1976         mutex_exit(&muxifier);
1977         *rvalp = linkp->li_lblk.l_index;
1978         netstack_rele(ss->ss_netstack);
1979         return (0);
1980 }
1981 
1982 int
1983 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1984 {
1985         int             ret;
1986         struct file     *fpdown;
1987 
1988         fpdown = getf(arg);
1989         ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1990         if (fpdown != NULL)
1991                 releasef(arg);
1992         return (ret);
1993 }
1994 
1995 /*
1996  * Unlink a multiplexor link. Stp is the controlling stream for the
1997  * link, and linkp points to the link's entry in the linkinfo list.
1998  * The muxifier lock must be held on entry and is dropped on exit.
1999  *
2000  * NOTE : Currently it is assumed that mux would process all the messages
2001  * sitting on it's queue before ACKing the UNLINK. It is the responsibility
2002  * of the mux to handle all the messages that arrive before UNLINK.
2003  * If the mux has to send down messages on its lower stream before
2004  * ACKing I_UNLINK, then it *should* know to handle messages even
2005  * after the UNLINK is acked (actually it should be able to handle till we
2006  * re-block the read side of the pass queue here). If the mux does not
2007  * open up the lower stream, any messages that arrive during UNLINK
2008  * will be put in the stream head. In the case of lower stream opening
2009  * up, some messages might land in the stream head depending on when
2010  * the message arrived and when the read side of the pass queue was
2011  * re-blocked.
2012  */
2013 int
2014 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2015     str_stack_t *ss)
2016 {
2017         struct strioctl strioc;
2018         struct stdata *stpdown;
2019         queue_t *rq, *wrq;
2020         queue_t *passq;
2021         syncq_t *passyncq;
2022         int error = 0;
2023         file_t *fpdown;
2024 
2025         ASSERT(MUTEX_HELD(&muxifier));
2026 
2027         stpdown = linkp->li_fpdown->f_vnode->v_stream;
2028 
2029         /*
2030          * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2031          */
2032         mutex_enter(&stpdown->sd_lock);
2033         stpdown->sd_flag |= STRPLUMB;
2034         mutex_exit(&stpdown->sd_lock);
2035 
2036         /*
2037          * Add passthru queue below lower mux. This will block
2038          * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2039          */
2040         passq = link_addpassthru(stpdown);
2041 
2042         if ((flag & LINKTYPEMASK) == LINKNORMAL)
2043                 strioc.ic_cmd = I_UNLINK;
2044         else
2045                 strioc.ic_cmd = I_PUNLINK;
2046         strioc.ic_timout = INFTIM;
2047         strioc.ic_len = sizeof (struct linkblk);
2048         strioc.ic_dp = (char *)&linkp->li_lblk;
2049 
2050         error = strdoioctl(stp, &strioc, FNATIVE,
2051             K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2052 
2053         /*
2054          * If there was an error and this is not called via strclose,
2055          * return to the user. Otherwise, pretend there was no error
2056          * and close the link.
2057          */
2058         if (error) {
2059                 if (flag & LINKCLOSE) {
2060                         cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2061                             "unlink ioctl, closing anyway (%d)\n", error);
2062                 } else {
2063                         link_rempassthru(passq);
2064                         mutex_enter(&stpdown->sd_lock);
2065                         stpdown->sd_flag &= ~STRPLUMB;
2066                         cv_broadcast(&stpdown->sd_monitor);
2067                         mutex_exit(&stpdown->sd_lock);
2068                         mutex_exit(&muxifier);
2069                         return (error);
2070                 }
2071         }
2072 
2073         mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2074         fpdown = linkp->li_fpdown;
2075         lbfree(linkp);
2076 
2077         /*
2078          * We go ahead and drop muxifier here--it's a nasty global lock that
2079          * can slow others down. It's okay to since attempts to mlink() this
2080          * stream will be stopped because STPLEX is still set in the stdata
2081          * structure, and munlink() is stopped because mux_rmvedge() and
2082          * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2083          * respectively.  Note that we defer the closef() of fpdown until
2084          * after we drop muxifier since strclose() can call munlinkall().
2085          */
2086         mutex_exit(&muxifier);
2087 
2088         wrq = stpdown->sd_wrq;
2089         rq = _RD(wrq);
2090 
2091         /*
2092          * Get rid of outstanding service procedure runs, before we make
2093          * it a stream head, since a stream head doesn't have any service
2094          * procedure.
2095          */
2096         disable_svc(rq);
2097         wait_svc(rq);
2098 
2099         /*
2100          * Since we don't disable the syncq for QPERMOD, we wait for whatever
2101          * is queued up to be finished. mux should take care that nothing is
2102          * send down to this queue. We should do it now as we're going to block
2103          * passyncq if it was unblocked.
2104          */
2105         if (wrq->q_flag & QPERMOD) {
2106                 syncq_t *sq = wrq->q_syncq;
2107 
2108                 mutex_enter(SQLOCK(sq));
2109                 while (wrq->q_sqflags & Q_SQQUEUED) {
2110                         sq->sq_flags |= SQ_WANTWAKEUP;
2111                         cv_wait(&sq->sq_wait, SQLOCK(sq));
2112                 }
2113                 mutex_exit(SQLOCK(sq));
2114         }
2115         passyncq = passq->q_syncq;
2116         if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2117 
2118                 syncq_t *sq, *outer;
2119 
2120                 /*
2121                  * Messages could be flowing from underneath. We will
2122                  * block the read side of the passq. This would be
2123                  * sufficient for QPAIR and QPERQ muxes to ensure
2124                  * that no data is flowing up into this queue
2125                  * and hence no thread active in this instance of
2126                  * lower mux. But for QPERMOD and QMTOUTPERIM there
2127                  * could be messages on the inner and outer/inner
2128                  * syncqs respectively. We will wait for them to drain.
2129                  * Because passq is blocked messages end up in the syncq
2130                  * And qfill_syncq could possibly end up setting QFULL
2131                  * which will access the rq->q_flag. Hence, we have to
2132                  * acquire the QLOCK in setq.
2133                  *
2134                  * XXX Messages can also flow from top into this
2135                  * queue though the unlink is over (Ex. some instance
2136                  * in putnext() called from top that has still not
2137                  * accessed this queue. And also putq(lowerq) ?).
2138                  * Solution : How about blocking the l_qtop queue ?
2139                  * Do we really care about such pure D_MP muxes ?
2140                  */
2141 
2142                 blocksq(passyncq, SQ_BLOCKED, 0);
2143 
2144                 sq = rq->q_syncq;
2145                 if ((outer = sq->sq_outer) != NULL) {
2146 
2147                         /*
2148                          * We have to just wait for the outer sq_count
2149                          * drop to zero. As this does not prevent new
2150                          * messages to enter the outer perimeter, this
2151                          * is subject to starvation.
2152                          *
2153                          * NOTE :Because of blocksq above, messages could
2154                          * be in the inner syncq only because of some
2155                          * thread holding the outer perimeter exclusively.
2156                          * Hence it would be sufficient to wait for the
2157                          * exclusive holder of the outer perimeter to drain
2158                          * the inner and outer syncqs. But we will not depend
2159                          * on this feature and hence check the inner syncqs
2160                          * separately.
2161                          */
2162                         wait_syncq(outer);
2163                 }
2164 
2165 
2166                 /*
2167                  * There could be messages destined for
2168                  * this queue. Let the exclusive holder
2169                  * drain it.
2170                  */
2171 
2172                 wait_syncq(sq);
2173                 ASSERT((rq->q_flag & QPERMOD) ||
2174                     ((rq->q_syncq->sq_head == NULL) &&
2175                     (_WR(rq)->q_syncq->sq_head == NULL)));
2176         }
2177 
2178         /*
2179          * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2180          * case as we don't disable its syncq or remove it off the syncq
2181          * service list.
2182          */
2183         if (rq->q_flag & QPERMOD) {
2184                 syncq_t *sq = rq->q_syncq;
2185 
2186                 mutex_enter(SQLOCK(sq));
2187                 while (rq->q_sqflags & Q_SQQUEUED) {
2188                         sq->sq_flags |= SQ_WANTWAKEUP;
2189                         cv_wait(&sq->sq_wait, SQLOCK(sq));
2190                 }
2191                 mutex_exit(SQLOCK(sq));
2192         }
2193 
2194         /*
2195          * flush_syncq changes states only when there are some messages to
2196          * free, i.e. when it returns non-zero value to return.
2197          */
2198         ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2199         ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2200 
2201         /*
2202          * Nobody else should know about this queue now.
2203          * If the mux did not process the messages before
2204          * acking the I_UNLINK, free them now.
2205          */
2206 
2207         flushq(rq, FLUSHALL);
2208         flushq(_WR(rq), FLUSHALL);
2209 
2210         /*
2211          * Convert the mux lower queue into a stream head queue.
2212          * Turn off STPLEX before we turn on the stream by removing the passq.
2213          */
2214         rq->q_ptr = wrq->q_ptr = stpdown;
2215         setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2216 
2217         ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2218         ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2219 
2220         enable_svc(rq);
2221 
2222         /*
2223          * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2224          * needs to be set to prevent reopen() of the stream - such reopen may
2225          * try to call non-existent pass queue open routine and panic.
2226          */
2227         mutex_enter(&stpdown->sd_lock);
2228         stpdown->sd_flag &= ~STPLEX;
2229         mutex_exit(&stpdown->sd_lock);
2230 
2231         ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2232             ((flag & LINKTYPEMASK) == LINKPERSIST));
2233 
2234         /* clean up the layered driver linkages */
2235         if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2236                 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2237         } else {
2238                 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2239         }
2240 
2241         link_rempassthru(passq);
2242 
2243         /*
2244          * Now all plumbing changes are finished and STRPLUMB is no
2245          * longer needed.
2246          */
2247         mutex_enter(&stpdown->sd_lock);
2248         stpdown->sd_flag &= ~STRPLUMB;
2249         cv_broadcast(&stpdown->sd_monitor);
2250         mutex_exit(&stpdown->sd_lock);
2251 
2252         (void) closef(fpdown);
2253         return (0);
2254 }
2255 
2256 /*
2257  * Unlink all multiplexor links for which stp is the controlling stream.
2258  * Return 0, or a non-zero errno on failure.
2259  */
2260 int
2261 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2262 {
2263         linkinfo_t *linkp;
2264         int error = 0;
2265 
2266         mutex_enter(&muxifier);
2267         while (linkp = findlinks(stp, 0, flag, ss)) {
2268                 /*
2269                  * munlink() releases the muxifier lock.
2270                  */
2271                 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2272                         return (error);
2273                 mutex_enter(&muxifier);
2274         }
2275         mutex_exit(&muxifier);
2276         return (0);
2277 }
2278 
2279 /*
2280  * A multiplexor link has been made. Add an
2281  * edge to the directed graph.
2282  */
2283 void
2284 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2285 {
2286         struct mux_node *np;
2287         struct mux_edge *ep;
2288         major_t upmaj;
2289         major_t lomaj;
2290 
2291         upmaj = getmajor(upstp->sd_vnode->v_rdev);
2292         lomaj = getmajor(lostp->sd_vnode->v_rdev);
2293         np = &ss->ss_mux_nodes[upmaj];
2294         if (np->mn_outp) {
2295                 ep = np->mn_outp;
2296                 while (ep->me_nextp)
2297                         ep = ep->me_nextp;
2298                 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2299                 ep = ep->me_nextp;
2300         } else {
2301                 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2302                 ep = np->mn_outp;
2303         }
2304         ep->me_nextp = NULL;
2305         ep->me_muxid = muxid;
2306         /*
2307          * Save the dev_t for the purposes of str_stack_shutdown.
2308          * str_stack_shutdown assumes that the device allows reopen, since
2309          * this dev_t is the one after any cloning by xx_open().
2310          * Would prefer finding the dev_t from before any cloning,
2311          * but specfs doesn't retain that.
2312          */
2313         ep->me_dev = upstp->sd_vnode->v_rdev;
2314         if (lostp->sd_vnode->v_type == VFIFO)
2315                 ep->me_nodep = NULL;
2316         else
2317                 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2318 }
2319 
2320 /*
2321  * A multiplexor link has been removed. Remove the
2322  * edge in the directed graph.
2323  */
2324 void
2325 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2326 {
2327         struct mux_node *np;
2328         struct mux_edge *ep;
2329         struct mux_edge *pep = NULL;
2330         major_t upmaj;
2331 
2332         upmaj = getmajor(upstp->sd_vnode->v_rdev);
2333         np = &ss->ss_mux_nodes[upmaj];
2334         ASSERT(np->mn_outp != NULL);
2335         ep = np->mn_outp;
2336         while (ep) {
2337                 if (ep->me_muxid == muxid) {
2338                         if (pep)
2339                                 pep->me_nextp = ep->me_nextp;
2340                         else
2341                                 np->mn_outp = ep->me_nextp;
2342                         kmem_free(ep, sizeof (struct mux_edge));
2343                         return;
2344                 }
2345                 pep = ep;
2346                 ep = ep->me_nextp;
2347         }
2348         ASSERT(0);      /* should not reach here */
2349 }
2350 
2351 /*
2352  * Translate the device flags (from conf.h) to the corresponding
2353  * qflag and sq_flag (type) values.
2354  */
2355 int
2356 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2357     uint32_t *sqtypep)
2358 {
2359         uint32_t qflag = 0;
2360         uint32_t sqtype = 0;
2361 
2362         if (devflag & _D_OLD)
2363                 goto bad;
2364 
2365         /* Inner perimeter presence and scope */
2366         switch (devflag & D_MTINNER_MASK) {
2367         case D_MP:
2368                 qflag |= QMTSAFE;
2369                 sqtype |= SQ_CI;
2370                 break;
2371         case D_MTPERQ|D_MP:
2372                 qflag |= QPERQ;
2373                 break;
2374         case D_MTQPAIR|D_MP:
2375                 qflag |= QPAIR;
2376                 break;
2377         case D_MTPERMOD|D_MP:
2378                 qflag |= QPERMOD;
2379                 break;
2380         default:
2381                 goto bad;
2382         }
2383 
2384         /* Outer perimeter */
2385         if (devflag & D_MTOUTPERIM) {
2386                 switch (devflag & D_MTINNER_MASK) {
2387                 case D_MP:
2388                 case D_MTPERQ|D_MP:
2389                 case D_MTQPAIR|D_MP:
2390                         break;
2391                 default:
2392                         goto bad;
2393                 }
2394                 qflag |= QMTOUTPERIM;
2395         }
2396 
2397         /* Inner perimeter modifiers */
2398         if (devflag & D_MTINNER_MOD) {
2399                 switch (devflag & D_MTINNER_MASK) {
2400                 case D_MP:
2401                         goto bad;
2402                 default:
2403                         break;
2404                 }
2405                 if (devflag & D_MTPUTSHARED)
2406                         sqtype |= SQ_CIPUT;
2407                 if (devflag & _D_MTOCSHARED) {
2408                         /*
2409                          * The code in putnext assumes that it has the
2410                          * highest concurrency by not checking sq_count.
2411                          * Thus _D_MTOCSHARED can only be supported when
2412                          * D_MTPUTSHARED is set.
2413                          */
2414                         if (!(devflag & D_MTPUTSHARED))
2415                                 goto bad;
2416                         sqtype |= SQ_CIOC;
2417                 }
2418                 if (devflag & _D_MTCBSHARED) {
2419                         /*
2420                          * The code in putnext assumes that it has the
2421                          * highest concurrency by not checking sq_count.
2422                          * Thus _D_MTCBSHARED can only be supported when
2423                          * D_MTPUTSHARED is set.
2424                          */
2425                         if (!(devflag & D_MTPUTSHARED))
2426                                 goto bad;
2427                         sqtype |= SQ_CICB;
2428                 }
2429                 if (devflag & _D_MTSVCSHARED) {
2430                         /*
2431                          * The code in putnext assumes that it has the
2432                          * highest concurrency by not checking sq_count.
2433                          * Thus _D_MTSVCSHARED can only be supported when
2434                          * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2435                          * supported only for QPERMOD.
2436                          */
2437                         if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2438                                 goto bad;
2439                         sqtype |= SQ_CISVC;
2440                 }
2441         }
2442 
2443         /* Default outer perimeter concurrency */
2444         sqtype |= SQ_CO;
2445 
2446         /* Outer perimeter modifiers */
2447         if (devflag & D_MTOCEXCL) {
2448                 if (!(devflag & D_MTOUTPERIM)) {
2449                         /* No outer perimeter */
2450                         goto bad;
2451                 }
2452                 sqtype &= ~SQ_COOC;
2453         }
2454 
2455         /* Synchronous Streams extended qinit structure */
2456         if (devflag & D_SYNCSTR)
2457                 qflag |= QSYNCSTR;
2458 
2459         /*
2460          * Private flag used by a transport module to indicate
2461          * to sockfs that it supports direct-access mode without
2462          * having to go through STREAMS.
2463          */
2464         if (devflag & _D_DIRECT) {
2465                 /* Reject unless the module is fully-MT (no perimeter) */
2466                 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2467                         goto bad;
2468                 qflag |= _QDIRECT;
2469         }
2470 
2471         *qflagp = qflag;
2472         *sqtypep = sqtype;
2473         return (0);
2474 
2475 bad:
2476         cmn_err(CE_WARN,
2477             "stropen: bad MT flags (0x%x) in driver '%s'",
2478             (int)(qflag & D_MTSAFETY_MASK),
2479             stp->st_rdinit->qi_minfo->mi_idname);
2480 
2481         return (EINVAL);
2482 }
2483 
2484 /*
2485  * Set the interface values for a pair of queues (qinit structure,
2486  * packet sizes, water marks).
2487  * setq assumes that the caller does not have a claim (entersq or claimq)
2488  * on the queue.
2489  */
2490 void
2491 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2492     perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2493 {
2494         queue_t *wq;
2495         syncq_t *sq, *outer;
2496 
2497         ASSERT(rq->q_flag & QREADR);
2498         ASSERT((qflag & QMT_TYPEMASK) != 0);
2499         IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2500 
2501         wq = _WR(rq);
2502         rq->q_qinfo = rinit;
2503         rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2504         rq->q_lowat = rinit->qi_minfo->mi_lowat;
2505         rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2506         rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2507         wq->q_qinfo = winit;
2508         wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2509         wq->q_lowat = winit->qi_minfo->mi_lowat;
2510         wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2511         wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2512 
2513         /* Remove old syncqs */
2514         sq = rq->q_syncq;
2515         outer = sq->sq_outer;
2516         if (outer != NULL) {
2517                 ASSERT(wq->q_syncq->sq_outer == outer);
2518                 outer_remove(outer, rq->q_syncq);
2519                 if (wq->q_syncq != rq->q_syncq)
2520                         outer_remove(outer, wq->q_syncq);
2521         }
2522         ASSERT(sq->sq_outer == NULL);
2523         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2524 
2525         if (sq != SQ(rq)) {
2526                 if (!(rq->q_flag & QPERMOD))
2527                         free_syncq(sq);
2528                 if (wq->q_syncq == rq->q_syncq)
2529                         wq->q_syncq = NULL;
2530                 rq->q_syncq = NULL;
2531         }
2532         if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2533             wq->q_syncq != SQ(rq)) {
2534                 free_syncq(wq->q_syncq);
2535                 wq->q_syncq = NULL;
2536         }
2537         ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2538             rq->q_syncq->sq_tail == NULL));
2539         ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2540             wq->q_syncq->sq_tail == NULL));
2541 
2542         if (!(rq->q_flag & QPERMOD) &&
2543             rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2544                 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2545                 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2546                     rq->q_syncq->sq_nciputctrl, 0);
2547                 ASSERT(ciputctrl_cache != NULL);
2548                 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2549                 rq->q_syncq->sq_ciputctrl = NULL;
2550                 rq->q_syncq->sq_nciputctrl = 0;
2551         }
2552 
2553         if (!(wq->q_flag & QPERMOD) &&
2554             wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2555                 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2556                 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2557                     wq->q_syncq->sq_nciputctrl, 0);
2558                 ASSERT(ciputctrl_cache != NULL);
2559                 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2560                 wq->q_syncq->sq_ciputctrl = NULL;
2561                 wq->q_syncq->sq_nciputctrl = 0;
2562         }
2563 
2564         sq = SQ(rq);
2565         ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2566         ASSERT(sq->sq_outer == NULL);
2567         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2568 
2569         /*
2570          * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2571          * bits in sq_flag based on the sqtype.
2572          */
2573         ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2574 
2575         rq->q_syncq = wq->q_syncq = sq;
2576         sq->sq_type = sqtype;
2577         sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2578 
2579         /*
2580          *  We are making sq_svcflags zero,
2581          *  resetting SQ_DISABLED in case it was set by
2582          *  wait_svc() in the munlink path.
2583          *
2584          */
2585         ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2586         sq->sq_svcflags = 0;
2587 
2588         /*
2589          * We need to acquire the lock here for the mlink and munlink case,
2590          * where canputnext, backenable, etc can access the q_flag.
2591          */
2592         if (lock_needed) {
2593                 mutex_enter(QLOCK(rq));
2594                 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2595                 mutex_exit(QLOCK(rq));
2596                 mutex_enter(QLOCK(wq));
2597                 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2598                 mutex_exit(QLOCK(wq));
2599         } else {
2600                 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2601                 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2602         }
2603 
2604         if (qflag & QPERQ) {
2605                 /* Allocate a separate syncq for the write side */
2606                 sq = new_syncq();
2607                 sq->sq_type = rq->q_syncq->sq_type;
2608                 sq->sq_flags = rq->q_syncq->sq_flags;
2609                 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2610                     sq->sq_oprev == NULL);
2611                 wq->q_syncq = sq;
2612         }
2613         if (qflag & QPERMOD) {
2614                 sq = dmp->dm_sq;
2615 
2616                 /*
2617                  * Assert that we do have an inner perimeter syncq and that it
2618                  * does not have an outer perimeter associated with it.
2619                  */
2620                 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2621                     sq->sq_oprev == NULL);
2622                 rq->q_syncq = wq->q_syncq = sq;
2623         }
2624         if (qflag & QMTOUTPERIM) {
2625                 outer = dmp->dm_sq;
2626 
2627                 ASSERT(outer->sq_outer == NULL);
2628                 outer_insert(outer, rq->q_syncq);
2629                 if (wq->q_syncq != rq->q_syncq)
2630                         outer_insert(outer, wq->q_syncq);
2631         }
2632         ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2633             (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2634         ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2635             (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2636         ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2637 
2638         /*
2639          * Initialize struio() types.
2640          */
2641         rq->q_struiot =
2642             (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2643         wq->q_struiot =
2644             (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2645 }
2646 
2647 perdm_t *
2648 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2649 {
2650         syncq_t *sq;
2651         perdm_t **pp;
2652         perdm_t *p;
2653         perdm_t *dmp;
2654 
2655         ASSERT(str != NULL);
2656         ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2657 
2658         rw_enter(&perdm_rwlock, RW_READER);
2659         for (p = perdm_list; p != NULL; p = p->dm_next) {
2660                 if (p->dm_str == str) {      /* found one */
2661                         atomic_inc_32(&(p->dm_ref));
2662                         rw_exit(&perdm_rwlock);
2663                         return (p);
2664                 }
2665         }
2666         rw_exit(&perdm_rwlock);
2667 
2668         sq = new_syncq();
2669         if (qflag & QPERMOD) {
2670                 sq->sq_type = sqtype | SQ_PERMOD;
2671                 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2672         } else {
2673                 ASSERT(qflag & QMTOUTPERIM);
2674                 sq->sq_onext = sq->sq_oprev = sq;
2675         }
2676 
2677         dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2678         dmp->dm_sq = sq;
2679         dmp->dm_str = str;
2680         dmp->dm_ref = 1;
2681         dmp->dm_next = NULL;
2682 
2683         rw_enter(&perdm_rwlock, RW_WRITER);
2684         for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2685                 if (p->dm_str == str) {      /* already present */
2686                         p->dm_ref++;
2687                         rw_exit(&perdm_rwlock);
2688                         free_syncq(sq);
2689                         kmem_free(dmp, sizeof (perdm_t));
2690                         return (p);
2691                 }
2692         }
2693 
2694         *pp = dmp;
2695         rw_exit(&perdm_rwlock);
2696         return (dmp);
2697 }
2698 
2699 void
2700 rele_dm(perdm_t *dmp)
2701 {
2702         perdm_t **pp;
2703         perdm_t *p;
2704 
2705         rw_enter(&perdm_rwlock, RW_WRITER);
2706         ASSERT(dmp->dm_ref > 0);
2707 
2708         if (--dmp->dm_ref > 0) {
2709                 rw_exit(&perdm_rwlock);
2710                 return;
2711         }
2712 
2713         for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2714                 if (p == dmp)
2715                         break;
2716         ASSERT(p == dmp);
2717         *pp = p->dm_next;
2718         rw_exit(&perdm_rwlock);
2719 
2720         /*
2721          * Wait for any background processing that relies on the
2722          * syncq to complete before it is freed.
2723          */
2724         wait_sq_svc(p->dm_sq);
2725         free_syncq(p->dm_sq);
2726         kmem_free(p, sizeof (perdm_t));
2727 }
2728 
2729 /*
2730  * Make a protocol message given control and data buffers.
2731  * n.b., this can block; be careful of what locks you hold when calling it.
2732  *
2733  * If sd_maxblk is less than *iosize this routine can fail part way through
2734  * (due to an allocation failure). In this case on return *iosize will contain
2735  * the amount that was consumed. Otherwise *iosize will not be modified
2736  * i.e. it will contain the amount that was consumed.
2737  */
2738 int
2739 strmakemsg(
2740         struct strbuf *mctl,
2741         ssize_t *iosize,
2742         struct uio *uiop,
2743         stdata_t *stp,
2744         int32_t flag,
2745         mblk_t **mpp)
2746 {
2747         mblk_t *mpctl = NULL;
2748         mblk_t *mpdata = NULL;
2749         int error;
2750 
2751         ASSERT(uiop != NULL);
2752 
2753         *mpp = NULL;
2754         /* Create control part, if any */
2755         if ((mctl != NULL) && (mctl->len >= 0)) {
2756                 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2757                 if (error)
2758                         return (error);
2759         }
2760         /* Create data part, if any */
2761         if (*iosize >= 0) {
2762                 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2763                 if (error) {
2764                         freemsg(mpctl);
2765                         return (error);
2766                 }
2767         }
2768         if (mpctl != NULL) {
2769                 if (mpdata != NULL)
2770                         linkb(mpctl, mpdata);
2771                 *mpp = mpctl;
2772         } else {
2773                 *mpp = mpdata;
2774         }
2775         return (0);
2776 }
2777 
2778 /*
2779  * Make the control part of a protocol message given a control buffer.
2780  * n.b., this can block; be careful of what locks you hold when calling it.
2781  */
2782 int
2783 strmakectl(
2784         struct strbuf *mctl,
2785         int32_t flag,
2786         int32_t fflag,
2787         mblk_t **mpp)
2788 {
2789         mblk_t *bp = NULL;
2790         unsigned char msgtype;
2791         int error = 0;
2792         cred_t *cr = CRED();
2793 
2794         /* We do not support interrupt threads using the stream head to send */
2795         ASSERT(cr != NULL);
2796 
2797         *mpp = NULL;
2798         /*
2799          * Create control part of message, if any.
2800          */
2801         if ((mctl != NULL) && (mctl->len >= 0)) {
2802                 caddr_t base;
2803                 int ctlcount;
2804                 int allocsz;
2805 
2806                 if (flag & RS_HIPRI)
2807                         msgtype = M_PCPROTO;
2808                 else
2809                         msgtype = M_PROTO;
2810 
2811                 ctlcount = mctl->len;
2812                 base = mctl->buf;
2813 
2814                 /*
2815                  * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2816                  * blocks by increasing the size to something more usable.
2817                  */
2818                 allocsz = MAX(ctlcount, 64);
2819 
2820                 /*
2821                  * Range checking has already been done; simply try
2822                  * to allocate a message block for the ctl part.
2823                  */
2824                 while ((bp = allocb_cred(allocsz, cr,
2825                     curproc->p_pid)) == NULL) {
2826                         if (fflag & (FNDELAY|FNONBLOCK))
2827                                 return (EAGAIN);
2828                         if (error = strwaitbuf(allocsz, BPRI_MED))
2829                                 return (error);
2830                 }
2831 
2832                 bp->b_datap->db_type = msgtype;
2833                 if (copyin(base, bp->b_wptr, ctlcount)) {
2834                         freeb(bp);
2835                         return (EFAULT);
2836                 }
2837                 bp->b_wptr += ctlcount;
2838         }
2839         *mpp = bp;
2840         return (0);
2841 }
2842 
2843 /*
2844  * Make a protocol message given data buffers.
2845  * n.b., this can block; be careful of what locks you hold when calling it.
2846  *
2847  * If sd_maxblk is less than *iosize this routine can fail part way through
2848  * (due to an allocation failure). In this case on return *iosize will contain
2849  * the amount that was consumed. Otherwise *iosize will not be modified
2850  * i.e. it will contain the amount that was consumed.
2851  */
2852 int
2853 strmakedata(
2854         ssize_t   *iosize,
2855         struct uio *uiop,
2856         stdata_t *stp,
2857         int32_t flag,
2858         mblk_t **mpp)
2859 {
2860         mblk_t *mp = NULL;
2861         mblk_t *bp;
2862         int wroff = (int)stp->sd_wroff;
2863         int tail_len = (int)stp->sd_tail;
2864         int extra = wroff + tail_len;
2865         int error = 0;
2866         ssize_t maxblk;
2867         ssize_t count = *iosize;
2868         cred_t *cr;
2869 
2870         *mpp = NULL;
2871         if (count < 0)
2872                 return (0);
2873 
2874         /* We do not support interrupt threads using the stream head to send */
2875         cr = CRED();
2876         ASSERT(cr != NULL);
2877 
2878         maxblk = stp->sd_maxblk;
2879         if (maxblk == INFPSZ)
2880                 maxblk = count;
2881 
2882         /*
2883          * Create data part of message, if any.
2884          */
2885         do {
2886                 ssize_t size;
2887                 dblk_t  *dp;
2888 
2889                 ASSERT(uiop);
2890 
2891                 size = MIN(count, maxblk);
2892 
2893                 while ((bp = allocb_cred(size + extra, cr,
2894                     curproc->p_pid)) == NULL) {
2895                         error = EAGAIN;
2896                         if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2897                             (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2898                                 if (count == *iosize) {
2899                                         freemsg(mp);
2900                                         return (error);
2901                                 } else {
2902                                         *iosize -= count;
2903                                         *mpp = mp;
2904                                         return (0);
2905                                 }
2906                         }
2907                 }
2908                 dp = bp->b_datap;
2909                 dp->db_cpid = curproc->p_pid;
2910                 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2911                 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2912 
2913                 if (flag & STRUIO_POSTPONE) {
2914                         /*
2915                          * Setup the stream uio portion of the
2916                          * dblk for subsequent use by struioget().
2917                          */
2918                         dp->db_struioflag = STRUIO_SPEC;
2919                         dp->db_cksumstart = 0;
2920                         dp->db_cksumstuff = 0;
2921                         dp->db_cksumend = size;
2922                         *(long long *)dp->db_struioun.data = 0ll;
2923                         bp->b_wptr += size;
2924                 } else {
2925                         if (stp->sd_copyflag & STRCOPYCACHED)
2926                                 uiop->uio_extflg |= UIO_COPY_CACHED;
2927 
2928                         if (size != 0) {
2929                                 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2930                                     uiop);
2931                                 if (error != 0) {
2932                                         freeb(bp);
2933                                         freemsg(mp);
2934                                         return (error);
2935                                 }
2936                         }
2937                         bp->b_wptr += size;
2938 
2939                         if (stp->sd_wputdatafunc != NULL) {
2940                                 mblk_t *newbp;
2941 
2942                                 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2943                                     bp, NULL, NULL, NULL, NULL);
2944                                 if (newbp == NULL) {
2945                                         freeb(bp);
2946                                         freemsg(mp);
2947                                         return (ECOMM);
2948                                 }
2949                                 bp = newbp;
2950                         }
2951                 }
2952 
2953                 count -= size;
2954 
2955                 if (mp == NULL)
2956                         mp = bp;
2957                 else
2958                         linkb(mp, bp);
2959         } while (count > 0);
2960 
2961         *mpp = mp;
2962         return (0);
2963 }
2964 
2965 /*
2966  * Wait for a buffer to become available. Return non-zero errno
2967  * if not able to wait, 0 if buffer is probably there.
2968  */
2969 int
2970 strwaitbuf(size_t size, int pri)
2971 {
2972         bufcall_id_t id;
2973 
2974         mutex_enter(&bcall_monitor);
2975         if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2976             &ttoproc(curthread)->p_flag_cv)) == 0) {
2977                 mutex_exit(&bcall_monitor);
2978                 return (ENOSR);
2979         }
2980         if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2981                 unbufcall(id);
2982                 mutex_exit(&bcall_monitor);
2983                 return (EINTR);
2984         }
2985         unbufcall(id);
2986         mutex_exit(&bcall_monitor);
2987         return (0);
2988 }
2989 
2990 /*
2991  * This function waits for a read or write event to happen on a stream.
2992  * fmode can specify FNDELAY and/or FNONBLOCK.
2993  * The timeout is in ms with -1 meaning infinite.
2994  * The flag values work as follows:
2995  *      READWAIT        Check for read side errors, send M_READ
2996  *      GETWAIT         Check for read side errors, no M_READ
2997  *      WRITEWAIT       Check for write side errors.
2998  *      NOINTR          Do not return error if nonblocking or timeout.
2999  *      STR_NOERROR     Ignore all errors except STPLEX.
3000  *      STR_NOSIG       Ignore/hold signals during the duration of the call.
3001  *      STR_PEEK        Pass through the strgeterr().
3002  */
3003 int
3004 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3005     int *done)
3006 {
3007         int slpflg, errs;
3008         int error;
3009         kcondvar_t *sleepon;
3010         mblk_t *mp;
3011         ssize_t *rd_count;
3012         clock_t rval;
3013 
3014         ASSERT(MUTEX_HELD(&stp->sd_lock));
3015         if ((flag & READWAIT) || (flag & GETWAIT)) {
3016                 slpflg = RSLEEP;
3017                 sleepon = &_RD(stp->sd_wrq)->q_wait;
3018                 errs = STRDERR|STPLEX;
3019         } else {
3020                 slpflg = WSLEEP;
3021                 sleepon = &stp->sd_wrq->q_wait;
3022                 errs = STWRERR|STRHUP|STPLEX;
3023         }
3024         if (flag & STR_NOERROR)
3025                 errs = STPLEX;
3026 
3027         if (stp->sd_wakeq & slpflg) {
3028                 /*
3029                  * A strwakeq() is pending, no need to sleep.
3030                  */
3031                 stp->sd_wakeq &= ~slpflg;
3032                 *done = 0;
3033                 return (0);
3034         }
3035 
3036         if (stp->sd_flag & errs) {
3037                 /*
3038                  * Check for errors before going to sleep since the
3039                  * caller might not have checked this while holding
3040                  * sd_lock.
3041                  */
3042                 error = strgeterr(stp, errs, (flag & STR_PEEK));
3043                 if (error != 0) {
3044                         *done = 1;
3045                         return (error);
3046                 }
3047         }
3048 
3049         /*
3050          * If any module downstream has requested read notification
3051          * by setting SNDMREAD flag using M_SETOPTS, send a message
3052          * down stream.
3053          */
3054         if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3055                 mutex_exit(&stp->sd_lock);
3056                 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3057                     (flag & STR_NOSIG), &error))) {
3058                         mutex_enter(&stp->sd_lock);
3059                         *done = 1;
3060                         return (error);
3061                 }
3062                 mp->b_datap->db_type = M_READ;
3063                 rd_count = (ssize_t *)mp->b_wptr;
3064                 *rd_count = count;
3065                 mp->b_wptr += sizeof (ssize_t);
3066                 /*
3067                  * Send the number of bytes requested by the
3068                  * read as the argument to M_READ.
3069                  */
3070                 stream_willservice(stp);
3071                 putnext(stp->sd_wrq, mp);
3072                 stream_runservice(stp);
3073                 mutex_enter(&stp->sd_lock);
3074 
3075                 /*
3076                  * If any data arrived due to inline processing
3077                  * of putnext(), don't sleep.
3078                  */
3079                 if (_RD(stp->sd_wrq)->q_first != NULL) {
3080                         *done = 0;
3081                         return (0);
3082                 }
3083         }
3084 
3085         if (fmode & (FNDELAY|FNONBLOCK)) {
3086                 if (!(flag & NOINTR))
3087                         error = EAGAIN;
3088                 else
3089                         error = 0;
3090                 *done = 1;
3091                 return (error);
3092         }
3093 
3094         stp->sd_flag |= slpflg;
3095         TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3096             "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3097             stp, flag, count, fmode, done);
3098 
3099         rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3100         if (rval > 0) {
3101                 /* EMPTY */
3102                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3103                     "strwaitq awakes(2):%X, %X, %X, %X, %X",
3104                     stp, flag, count, fmode, done);
3105         } else if (rval == 0) {
3106                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3107                     "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3108                     stp, flag, count, fmode, done);
3109                 stp->sd_flag &= ~slpflg;
3110                 cv_broadcast(sleepon);
3111                 if (!(flag & NOINTR))
3112                         error = EINTR;
3113                 else
3114                         error = 0;
3115                 *done = 1;
3116                 return (error);
3117         } else {
3118                 /* timeout */
3119                 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3120                     "strwaitq timeout:%p, %X, %lX, %X, %p",
3121                     stp, flag, count, fmode, done);
3122                 *done = 1;
3123                 if (!(flag & NOINTR))
3124                         return (ETIME);
3125                 else
3126                         return (0);
3127         }
3128         /*
3129          * If the caller implements delayed errors (i.e. queued after data)
3130          * we can not check for errors here since data as well as an
3131          * error might have arrived at the stream head. We return to
3132          * have the caller check the read queue before checking for errors.
3133          */
3134         if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3135                 error = strgeterr(stp, errs, (flag & STR_PEEK));
3136                 if (error != 0) {
3137                         *done = 1;
3138                         return (error);
3139                 }
3140         }
3141         *done = 0;
3142         return (0);
3143 }
3144 
3145 /*
3146  * Perform job control discipline access checks.
3147  * Return 0 for success and the errno for failure.
3148  */
3149 
3150 #define cantsend(p, t, sig) \
3151         (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3152 
3153 int
3154 straccess(struct stdata *stp, enum jcaccess mode)
3155 {
3156         extern kcondvar_t lbolt_cv;     /* XXX: should be in a header file */
3157         kthread_t *t = curthread;
3158         proc_t *p = ttoproc(t);
3159         sess_t *sp;
3160 
3161         ASSERT(mutex_owned(&stp->sd_lock));
3162 
3163         if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3164                 return (0);
3165 
3166         mutex_enter(&p->p_lock);         /* protects p_pgidp */
3167 
3168         for (;;) {
3169                 mutex_enter(&p->p_splock);       /* protects p->p_sessp */
3170                 sp = p->p_sessp;
3171                 mutex_enter(&sp->s_lock);        /* protects sp->* */
3172 
3173                 /*
3174                  * If this is not the calling process's controlling terminal
3175                  * or if the calling process is already in the foreground
3176                  * then allow access.
3177                  */
3178                 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3179                     p->p_pgidp == stp->sd_pgidp) {
3180                         mutex_exit(&sp->s_lock);
3181                         mutex_exit(&p->p_splock);
3182                         mutex_exit(&p->p_lock);
3183                         return (0);
3184                 }
3185 
3186                 /*
3187                  * Check to see if controlling terminal has been deallocated.
3188                  */
3189                 if (sp->s_vp == NULL) {
3190                         if (!cantsend(p, t, SIGHUP))
3191                                 sigtoproc(p, t, SIGHUP);
3192                         mutex_exit(&sp->s_lock);
3193                         mutex_exit(&p->p_splock);
3194                         mutex_exit(&p->p_lock);
3195                         return (EIO);
3196                 }
3197 
3198                 mutex_exit(&sp->s_lock);
3199                 mutex_exit(&p->p_splock);
3200 
3201                 if (mode == JCGETP) {
3202                         mutex_exit(&p->p_lock);
3203                         return (0);
3204                 }
3205 
3206                 if (mode == JCREAD) {
3207                         if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3208                                 mutex_exit(&p->p_lock);
3209                                 return (EIO);
3210                         }
3211                         mutex_exit(&p->p_lock);
3212                         mutex_exit(&stp->sd_lock);
3213                         pgsignal(p->p_pgidp, SIGTTIN);
3214                         mutex_enter(&stp->sd_lock);
3215                         mutex_enter(&p->p_lock);
3216                 } else {  /* mode == JCWRITE or JCSETP */
3217                         if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3218                             cantsend(p, t, SIGTTOU)) {
3219                                 mutex_exit(&p->p_lock);
3220                                 return (0);
3221                         }
3222                         if (p->p_detached) {
3223                                 mutex_exit(&p->p_lock);
3224                                 return (EIO);
3225                         }
3226                         mutex_exit(&p->p_lock);
3227                         mutex_exit(&stp->sd_lock);
3228                         pgsignal(p->p_pgidp, SIGTTOU);
3229                         mutex_enter(&stp->sd_lock);
3230                         mutex_enter(&p->p_lock);
3231                 }
3232 
3233                 /*
3234                  * We call cv_wait_sig_swap() to cause the appropriate
3235                  * action for the jobcontrol signal to take place.
3236                  * If the signal is being caught, we will take the
3237                  * EINTR error return.  Otherwise, the default action
3238                  * of causing the process to stop will take place.
3239                  * In this case, we rely on the periodic cv_broadcast() on
3240                  * &lbolt_cv to wake us up to loop around and test again.
3241                  * We can't get here if the signal is ignored or
3242                  * if the current thread is blocking the signal.
3243                  */
3244                 mutex_exit(&stp->sd_lock);
3245                 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3246                         mutex_exit(&p->p_lock);
3247                         mutex_enter(&stp->sd_lock);
3248                         return (EINTR);
3249                 }
3250                 mutex_exit(&p->p_lock);
3251                 mutex_enter(&stp->sd_lock);
3252                 mutex_enter(&p->p_lock);
3253         }
3254 }
3255 
3256 /*
3257  * Return size of message of block type (bp->b_datap->db_type)
3258  */
3259 size_t
3260 xmsgsize(mblk_t *bp)
3261 {
3262         unsigned char type;
3263         size_t count = 0;
3264 
3265         type = bp->b_datap->db_type;
3266 
3267         for (; bp; bp = bp->b_cont) {
3268                 if (type != bp->b_datap->db_type)
3269                         break;
3270                 ASSERT(bp->b_wptr >= bp->b_rptr);
3271                 count += bp->b_wptr - bp->b_rptr;
3272         }
3273         return (count);
3274 }
3275 
3276 /*
3277  * Allocate a stream head.
3278  */
3279 struct stdata *
3280 shalloc(queue_t *qp)
3281 {
3282         stdata_t *stp;
3283 
3284         stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3285 
3286         stp->sd_wrq = _WR(qp);
3287         stp->sd_strtab = NULL;
3288         stp->sd_iocid = 0;
3289         stp->sd_mate = NULL;
3290         stp->sd_freezer = NULL;
3291         stp->sd_refcnt = 0;
3292         stp->sd_wakeq = 0;
3293         stp->sd_anchor = 0;
3294         stp->sd_struiowrq = NULL;
3295         stp->sd_struiordq = NULL;
3296         stp->sd_struiodnak = 0;
3297         stp->sd_struionak = NULL;
3298         stp->sd_t_audit_data = NULL;
3299         stp->sd_rput_opt = 0;
3300         stp->sd_wput_opt = 0;
3301         stp->sd_read_opt = 0;
3302         stp->sd_rprotofunc = strrput_proto;
3303         stp->sd_rmiscfunc = strrput_misc;
3304         stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3305         stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3306         stp->sd_ciputctrl = NULL;
3307         stp->sd_nciputctrl = 0;
3308         stp->sd_qhead = NULL;
3309         stp->sd_qtail = NULL;
3310         stp->sd_servid = NULL;
3311         stp->sd_nqueues = 0;
3312         stp->sd_svcflags = 0;
3313         stp->sd_copyflag = 0;
3314 
3315         return (stp);
3316 }
3317 
3318 /*
3319  * Free a stream head.
3320  */
3321 void
3322 shfree(stdata_t *stp)
3323 {
3324         pid_node_t *pn;
3325 
3326         ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3327 
3328         stp->sd_wrq = NULL;
3329 
3330         mutex_enter(&stp->sd_qlock);
3331         while (stp->sd_svcflags & STRS_SCHEDULED) {
3332                 STRSTAT(strwaits);
3333                 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3334         }
3335         mutex_exit(&stp->sd_qlock);
3336 
3337         if (stp->sd_ciputctrl != NULL) {
3338                 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3339                 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3340                     stp->sd_nciputctrl, 0);
3341                 ASSERT(ciputctrl_cache != NULL);
3342                 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3343                 stp->sd_ciputctrl = NULL;
3344                 stp->sd_nciputctrl = 0;
3345         }
3346         ASSERT(stp->sd_qhead == NULL);
3347         ASSERT(stp->sd_qtail == NULL);
3348         ASSERT(stp->sd_nqueues == 0);
3349 
3350         mutex_enter(&stp->sd_pid_tree_lock);
3351         while ((pn = avl_first(&stp->sd_pid_tree)) != NULL) {
3352                 avl_remove(&stp->sd_pid_tree, pn);
3353                 kmem_free(pn, sizeof (*pn));
3354         }
3355         mutex_exit(&stp->sd_pid_tree_lock);
3356 
3357         kmem_cache_free(stream_head_cache, stp);
3358 }
3359 
3360 void
3361 sh_insert_pid(struct stdata *stp, pid_t pid)
3362 {
3363         pid_node_t *pn, lookup_pn;
3364         avl_index_t idx_pn;
3365 
3366         lookup_pn.pn_pid = pid;
3367         mutex_enter(&stp->sd_pid_tree_lock);
3368         pn = avl_find(&stp->sd_pid_tree, &lookup_pn, &idx_pn);
3369 
3370         if (pn != NULL) {
3371                 pn->pn_count++;
3372         } else {
3373                 pn = kmem_zalloc(sizeof (*pn), KM_SLEEP);
3374                 pn->pn_pid = pid;
3375                 pn->pn_count = 1;
3376                 avl_insert(&stp->sd_pid_tree, pn, idx_pn);
3377         }
3378         mutex_exit(&stp->sd_pid_tree_lock);
3379 }
3380 
3381 void
3382 sh_remove_pid(struct stdata *stp, pid_t pid)
3383 {
3384         pid_node_t *pn, lookup_pn;
3385 
3386         lookup_pn.pn_pid = pid;
3387         mutex_enter(&stp->sd_pid_tree_lock);
3388         pn = avl_find(&stp->sd_pid_tree, &lookup_pn, NULL);
3389 
3390         if (pn != NULL) {
3391                 if (pn->pn_count > 1) {
3392                         pn->pn_count--;
3393                 } else {
3394                         avl_remove(&stp->sd_pid_tree, pn);
3395                         kmem_free(pn, sizeof (*pn));
3396                 }
3397         }
3398         mutex_exit(&stp->sd_pid_tree_lock);
3399 }
3400 
3401 mblk_t *
3402 sh_get_pid_mblk(struct stdata *stp)
3403 {
3404         mblk_t *mblk;
3405         ulong_t sz, n;
3406         pid_t *pids;
3407         pid_node_t *pn;
3408         conn_pid_info_t *cpi;
3409 
3410         mutex_enter(&stp->sd_pid_tree_lock);
3411 
3412         n = avl_numnodes(&stp->sd_pid_tree);
3413         sz = sizeof (conn_pid_info_t);
3414         sz += (n > 1) ? ((n - 1) * sizeof (pid_t)) : 0;
3415         if ((mblk = allocb(sz, BPRI_HI)) == NULL) {
3416                 mutex_exit(&stp->sd_pid_tree_lock);
3417                 return (NULL);
3418         }
3419         mblk->b_wptr += sz;
3420         cpi = (conn_pid_info_t *)mblk->b_datap->db_base;
3421         cpi->cpi_magic = CONN_PID_INFO_MGC;
3422         cpi->cpi_contents = CONN_PID_INFO_XTI;
3423         cpi->cpi_pids_cnt = n;
3424         cpi->cpi_tot_size = sz;
3425         cpi->cpi_pids[0] = 0;
3426 
3427         if (cpi->cpi_pids_cnt > 0) {
3428                 pids = cpi->cpi_pids;
3429                 for (pn = avl_first(&stp->sd_pid_tree); pn != NULL;
3430                     pids++, pn = AVL_NEXT(&stp->sd_pid_tree, pn))
3431                         *pids = pn->pn_pid;
3432         }
3433         mutex_exit(&stp->sd_pid_tree_lock);
3434         return (mblk);
3435 }
3436 
3437 /*
3438  * Allocate a pair of queues and a syncq for the pair
3439  */
3440 queue_t *
3441 allocq(void)
3442 {
3443         queinfo_t *qip;
3444         queue_t *qp, *wqp;
3445         syncq_t *sq;
3446 
3447         qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3448 
3449         qp = &qip->qu_rqueue;
3450         wqp = &qip->qu_wqueue;
3451         sq = &qip->qu_syncq;
3452 
3453         qp->q_last   = NULL;
3454         qp->q_next   = NULL;
3455         qp->q_ptr    = NULL;
3456         qp->q_flag   = QUSE | QREADR;
3457         qp->q_bandp  = NULL;
3458         qp->q_stream = NULL;
3459         qp->q_syncq  = sq;
3460         qp->q_nband  = 0;
3461         qp->q_nfsrv  = NULL;
3462         qp->q_draining       = 0;
3463         qp->q_syncqmsgs      = 0;
3464         qp->q_spri   = 0;
3465         qp->q_qtstamp        = 0;
3466         qp->q_sqtstamp       = 0;
3467         qp->q_fp     = NULL;
3468 
3469         wqp->q_last  = NULL;
3470         wqp->q_next  = NULL;
3471         wqp->q_ptr   = NULL;
3472         wqp->q_flag  = QUSE;
3473         wqp->q_bandp = NULL;
3474         wqp->q_stream        = NULL;
3475         wqp->q_syncq = sq;
3476         wqp->q_nband = 0;
3477         wqp->q_nfsrv = NULL;
3478         wqp->q_draining      = 0;
3479         wqp->q_syncqmsgs = 0;
3480         wqp->q_qtstamp       = 0;
3481         wqp->q_sqtstamp      = 0;
3482         wqp->q_spri  = 0;
3483 
3484         sq->sq_count = 0;
3485         sq->sq_rmqcount      = 0;
3486         sq->sq_flags = 0;
3487         sq->sq_type  = 0;
3488         sq->sq_callbflags = 0;
3489         sq->sq_cancelid      = 0;
3490         sq->sq_ciputctrl = NULL;
3491         sq->sq_nciputctrl = 0;
3492         sq->sq_needexcl = 0;
3493         sq->sq_svcflags = 0;
3494 
3495         return (qp);
3496 }
3497 
3498 /*
3499  * Free a pair of queues and the "attached" syncq.
3500  * Discard any messages left on the syncq(s), remove the syncq(s) from the
3501  * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3502  */
3503 void
3504 freeq(queue_t *qp)
3505 {
3506         qband_t *qbp, *nqbp;
3507         syncq_t *sq, *outer;
3508         queue_t *wqp = _WR(qp);
3509 
3510         ASSERT(qp->q_flag & QREADR);
3511 
3512         /*
3513          * If a previously dispatched taskq job is scheduled to run
3514          * sync_service() or a service routine is scheduled for the
3515          * queues about to be freed, wait here until all service is
3516          * done on the queue and all associated queues and syncqs.
3517          */
3518         wait_svc(qp);
3519 
3520         (void) flush_syncq(qp->q_syncq, qp);
3521         (void) flush_syncq(wqp->q_syncq, wqp);
3522         ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3523 
3524         /*
3525          * Flush the queues before q_next is set to NULL This is needed
3526          * in order to backenable any downstream queue before we go away.
3527          * Note: we are already removed from the stream so that the
3528          * backenabling will not cause any messages to be delivered to our
3529          * put procedures.
3530          */
3531         flushq(qp, FLUSHALL);
3532         flushq(wqp, FLUSHALL);
3533 
3534         /* Tidy up - removeq only does a half-remove from stream */
3535         qp->q_next = wqp->q_next = NULL;
3536         ASSERT(!(qp->q_flag & QENAB));
3537         ASSERT(!(wqp->q_flag & QENAB));
3538 
3539         outer = qp->q_syncq->sq_outer;
3540         if (outer != NULL) {
3541                 outer_remove(outer, qp->q_syncq);
3542                 if (wqp->q_syncq != qp->q_syncq)
3543                         outer_remove(outer, wqp->q_syncq);
3544         }
3545         /*
3546          * Free any syncqs that are outside what allocq returned.
3547          */
3548         if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3549                 free_syncq(qp->q_syncq);
3550         if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3551                 free_syncq(wqp->q_syncq);
3552 
3553         ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3554         ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3555         ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3556         ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3557         sq = SQ(qp);
3558         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3559         ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3560         ASSERT(sq->sq_outer == NULL);
3561         ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3562         ASSERT(sq->sq_callbpend == NULL);
3563         ASSERT(sq->sq_needexcl == 0);
3564 
3565         if (sq->sq_ciputctrl != NULL) {
3566                 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3567                 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3568                     sq->sq_nciputctrl, 0);
3569                 ASSERT(ciputctrl_cache != NULL);
3570                 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3571                 sq->sq_ciputctrl = NULL;
3572                 sq->sq_nciputctrl = 0;
3573         }
3574 
3575         ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3576         ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3577         ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3578 
3579         qp->q_flag &= ~QUSE;
3580         wqp->q_flag &= ~QUSE;
3581 
3582         /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3583         /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3584 
3585         qbp = qp->q_bandp;
3586         while (qbp) {
3587                 nqbp = qbp->qb_next;
3588                 freeband(qbp);
3589                 qbp = nqbp;
3590         }
3591         qbp = wqp->q_bandp;
3592         while (qbp) {
3593                 nqbp = qbp->qb_next;
3594                 freeband(qbp);
3595                 qbp = nqbp;
3596         }
3597         kmem_cache_free(queue_cache, qp);
3598 }
3599 
3600 /*
3601  * Allocate a qband structure.
3602  */
3603 qband_t *
3604 allocband(void)
3605 {
3606         qband_t *qbp;
3607 
3608         qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3609         if (qbp == NULL)
3610                 return (NULL);
3611 
3612         qbp->qb_next = NULL;
3613         qbp->qb_count        = 0;
3614         qbp->qb_mblkcnt      = 0;
3615         qbp->qb_first        = NULL;
3616         qbp->qb_last = NULL;
3617         qbp->qb_flag = 0;
3618 
3619         return (qbp);
3620 }
3621 
3622 /*
3623  * Free a qband structure.
3624  */
3625 void
3626 freeband(qband_t *qbp)
3627 {
3628         kmem_cache_free(qband_cache, qbp);
3629 }
3630 
3631 /*
3632  * Just like putnextctl(9F), except that allocb_wait() is used.
3633  *
3634  * Consolidation Private, and of course only callable from the stream head or
3635  * routines that may block.
3636  */
3637 int
3638 putnextctl_wait(queue_t *q, int type)
3639 {
3640         mblk_t *bp;
3641         int error;
3642 
3643         if ((datamsg(type) && (type != M_DELAY)) ||
3644             (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3645                 return (0);
3646 
3647         bp->b_datap->db_type = (unsigned char)type;
3648         putnext(q, bp);
3649         return (1);
3650 }
3651 
3652 /*
3653  * Run any possible bufcalls.
3654  */
3655 void
3656 runbufcalls(void)
3657 {
3658         strbufcall_t *bcp;
3659 
3660         mutex_enter(&bcall_monitor);
3661         mutex_enter(&strbcall_lock);
3662 
3663         if (strbcalls.bc_head) {
3664                 size_t count;
3665                 int nevent;
3666 
3667                 /*
3668                  * count how many events are on the list
3669                  * now so we can check to avoid looping
3670                  * in low memory situations
3671                  */
3672                 nevent = 0;
3673                 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3674                         nevent++;
3675 
3676                 /*
3677                  * get estimate of available memory from kmem_avail().
3678                  * awake all bufcall functions waiting for
3679                  * memory whose request could be satisfied
3680                  * by 'count' memory and let 'em fight for it.
3681                  */
3682                 count = kmem_avail();
3683                 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3684                         STRSTAT(bufcalls);
3685                         --nevent;
3686                         if (bcp->bc_size <= count) {
3687                                 bcp->bc_executor = curthread;
3688                                 mutex_exit(&strbcall_lock);
3689                                 (*bcp->bc_func)(bcp->bc_arg);
3690                                 mutex_enter(&strbcall_lock);
3691                                 bcp->bc_executor = NULL;
3692                                 cv_broadcast(&bcall_cv);
3693                                 strbcalls.bc_head = bcp->bc_next;
3694                                 kmem_free(bcp, sizeof (strbufcall_t));
3695                         } else {
3696                                 /*
3697                                  * too big, try again later - note
3698                                  * that nevent was decremented above
3699                                  * so we won't retry this one on this
3700                                  * iteration of the loop
3701                                  */
3702                                 if (bcp->bc_next != NULL) {
3703                                         strbcalls.bc_head = bcp->bc_next;
3704                                         bcp->bc_next = NULL;
3705                                         strbcalls.bc_tail->bc_next = bcp;
3706                                         strbcalls.bc_tail = bcp;
3707                                 }
3708                         }
3709                 }
3710                 if (strbcalls.bc_head == NULL)
3711                         strbcalls.bc_tail = NULL;
3712         }
3713 
3714         mutex_exit(&strbcall_lock);
3715         mutex_exit(&bcall_monitor);
3716 }
3717 
3718 
3719 /*
3720  * Actually run queue's service routine.
3721  */
3722 static void
3723 runservice(queue_t *q)
3724 {
3725         qband_t *qbp;
3726 
3727         ASSERT(q->q_qinfo->qi_srvp);
3728 again:
3729         entersq(q->q_syncq, SQ_SVC);
3730         TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3731             "runservice starts:%p", q);
3732 
3733         if (!(q->q_flag & QWCLOSE))
3734                 (*q->q_qinfo->qi_srvp)(q);
3735 
3736         TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3737             "runservice ends:(%p)", q);
3738 
3739         leavesq(q->q_syncq, SQ_SVC);
3740 
3741         mutex_enter(QLOCK(q));
3742         if (q->q_flag & QENAB) {
3743                 q->q_flag &= ~QENAB;
3744                 mutex_exit(QLOCK(q));
3745                 goto again;
3746         }
3747         q->q_flag &= ~QINSERVICE;
3748         q->q_flag &= ~QBACK;
3749         for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3750                 qbp->qb_flag &= ~QB_BACK;
3751         /*
3752          * Wakeup thread waiting for the service procedure
3753          * to be run (strclose and qdetach).
3754          */
3755         cv_broadcast(&q->q_wait);
3756 
3757         mutex_exit(QLOCK(q));
3758 }
3759 
3760 /*
3761  * Background processing of bufcalls.
3762  */
3763 void
3764 streams_bufcall_service(void)
3765 {
3766         callb_cpr_t     cprinfo;
3767 
3768         CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3769             "streams_bufcall_service");
3770 
3771         mutex_enter(&strbcall_lock);
3772 
3773         for (;;) {
3774                 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3775                         mutex_exit(&strbcall_lock);
3776                         runbufcalls();
3777                         mutex_enter(&strbcall_lock);
3778                 }
3779                 if (strbcalls.bc_head != NULL) {
3780                         STRSTAT(bcwaits);
3781                         /* Wait for memory to become available */
3782                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3783                         (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3784                             SEC_TO_TICK(60), TR_CLOCK_TICK);
3785                         CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3786                 }
3787 
3788                 /* Wait for new work to arrive */
3789                 if (strbcalls.bc_head == NULL) {
3790                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3791                         cv_wait(&strbcall_cv, &strbcall_lock);
3792                         CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3793                 }
3794         }
3795 }
3796 
3797 /*
3798  * Background processing of streams background tasks which failed
3799  * taskq_dispatch.
3800  */
3801 static void
3802 streams_qbkgrnd_service(void)
3803 {
3804         callb_cpr_t cprinfo;
3805         queue_t *q;
3806 
3807         CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3808             "streams_bkgrnd_service");
3809 
3810         mutex_enter(&service_queue);
3811 
3812         for (;;) {
3813                 /*
3814                  * Wait for work to arrive.
3815                  */
3816                 while ((freebs_list == NULL) && (qhead == NULL)) {
3817                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3818                         cv_wait(&services_to_run, &service_queue);
3819                         CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3820                 }
3821                 /*
3822                  * Handle all pending freebs requests to free memory.
3823                  */
3824                 while (freebs_list != NULL) {
3825                         mblk_t *mp = freebs_list;
3826                         freebs_list = mp->b_next;
3827                         mutex_exit(&service_queue);
3828                         mblk_free(mp);
3829                         mutex_enter(&service_queue);
3830                 }
3831                 /*
3832                  * Run pending queues.
3833                  */
3834                 while (qhead != NULL) {
3835                         DQ(q, qhead, qtail, q_link);
3836                         ASSERT(q != NULL);
3837                         mutex_exit(&service_queue);
3838                         queue_service(q);
3839                         mutex_enter(&service_queue);
3840                 }
3841                 ASSERT(qhead == NULL && qtail == NULL);
3842         }
3843 }
3844 
3845 /*
3846  * Background processing of streams background tasks which failed
3847  * taskq_dispatch.
3848  */
3849 static void
3850 streams_sqbkgrnd_service(void)
3851 {
3852         callb_cpr_t cprinfo;
3853         syncq_t *sq;
3854 
3855         CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3856             "streams_sqbkgrnd_service");
3857 
3858         mutex_enter(&service_queue);
3859 
3860         for (;;) {
3861                 /*
3862                  * Wait for work to arrive.
3863                  */
3864                 while (sqhead == NULL) {
3865                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
3866                         cv_wait(&syncqs_to_run, &service_queue);
3867                         CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3868                 }
3869 
3870                 /*
3871                  * Run pending syncqs.
3872                  */
3873                 while (sqhead != NULL) {
3874                         DQ(sq, sqhead, sqtail, sq_next);
3875                         ASSERT(sq != NULL);
3876                         ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3877                         mutex_exit(&service_queue);
3878                         syncq_service(sq);
3879                         mutex_enter(&service_queue);
3880                 }
3881         }
3882 }
3883 
3884 /*
3885  * Disable the syncq and wait for background syncq processing to complete.
3886  * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3887  * list.
3888  */
3889 void
3890 wait_sq_svc(syncq_t *sq)
3891 {
3892         mutex_enter(SQLOCK(sq));
3893         sq->sq_svcflags |= SQ_DISABLED;
3894         if (sq->sq_svcflags & SQ_BGTHREAD) {
3895                 syncq_t *sq_chase;
3896                 syncq_t *sq_curr;
3897                 int removed;
3898 
3899                 ASSERT(sq->sq_servcount == 1);
3900                 mutex_enter(&service_queue);
3901                 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3902                 mutex_exit(&service_queue);
3903                 if (removed) {
3904                         sq->sq_svcflags &= ~SQ_BGTHREAD;
3905                         sq->sq_servcount = 0;
3906                         STRSTAT(sqremoved);
3907                         goto done;
3908                 }
3909         }
3910         while (sq->sq_servcount != 0) {
3911                 sq->sq_flags |= SQ_WANTWAKEUP;
3912                 cv_wait(&sq->sq_wait, SQLOCK(sq));
3913         }
3914 done:
3915         mutex_exit(SQLOCK(sq));
3916 }
3917 
3918 /*
3919  * Put a syncq on the list of syncq's to be serviced by the sqthread.
3920  * Add the argument to the end of the sqhead list and set the flag
3921  * indicating this syncq has been enabled.  If it has already been
3922  * enabled, don't do anything.
3923  * This routine assumes that SQLOCK is held.
3924  * NOTE that the lock order is to have the SQLOCK first,
3925  * so if the service_syncq lock is held, we need to release it
3926  * before acquiring the SQLOCK (mostly relevant for the background
3927  * thread, and this seems to be common among the STREAMS global locks).
3928  * Note that the sq_svcflags are protected by the SQLOCK.
3929  */
3930 void
3931 sqenable(syncq_t *sq)
3932 {
3933         /*
3934          * This is probably not important except for where I believe it
3935          * is being called.  At that point, it should be held (and it
3936          * is a pain to release it just for this routine, so don't do
3937          * it).
3938          */
3939         ASSERT(MUTEX_HELD(SQLOCK(sq)));
3940 
3941         IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3942         IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3943 
3944         /*
3945          * Do not put on list if background thread is scheduled or
3946          * syncq is disabled.
3947          */
3948         if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3949                 return;
3950 
3951         /*
3952          * Check whether we should enable sq at all.
3953          * Non PERMOD syncqs may be drained by at most one thread.
3954          * PERMOD syncqs may be drained by several threads but we limit the
3955          * total amount to the lesser of
3956          *      Number of queues on the squeue and
3957          *      Number of CPUs.
3958          */
3959         if (sq->sq_servcount != 0) {
3960                 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3961                     (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3962                         STRSTAT(sqtoomany);
3963                         return;
3964                 }
3965         }
3966 
3967         sq->sq_tstamp = ddi_get_lbolt();
3968         STRSTAT(sqenables);
3969 
3970         /* Attempt a taskq dispatch */
3971         sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3972             (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3973         if (sq->sq_servid != NULL) {
3974                 sq->sq_servcount++;
3975                 return;
3976         }
3977 
3978         /*
3979          * This taskq dispatch failed, but a previous one may have succeeded.
3980          * Don't try to schedule on the background thread whilst there is
3981          * outstanding taskq processing.
3982          */
3983         if (sq->sq_servcount != 0)
3984                 return;
3985 
3986         /*
3987          * System is low on resources and can't perform a non-sleeping
3988          * dispatch. Schedule the syncq for a background thread and mark the
3989          * syncq to avoid any further taskq dispatch attempts.
3990          */
3991         mutex_enter(&service_queue);
3992         STRSTAT(taskqfails);
3993         ENQUEUE(sq, sqhead, sqtail, sq_next);
3994         sq->sq_svcflags |= SQ_BGTHREAD;
3995         sq->sq_servcount = 1;
3996         cv_signal(&syncqs_to_run);
3997         mutex_exit(&service_queue);
3998 }
3999 
4000 /*
4001  * Note: fifo_close() depends on the mblk_t on the queue being freed
4002  * asynchronously. The asynchronous freeing of messages breaks the
4003  * recursive call chain of fifo_close() while there are I_SENDFD type of
4004  * messages referring to other file pointers on the queue. Then when
4005  * closing pipes it can avoid stack overflow in case of daisy-chained
4006  * pipes, and also avoid deadlock in case of fifonode_t pairs (which
4007  * share the same fifolock_t).
4008  *
4009  * No need to kpreempt_disable to access cpu_seqid.  If we migrate and
4010  * the esb queue does not match the new CPU, that is OK.
4011  */
4012 void
4013 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
4014 {
4015         int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
4016         esb_queue_t *eqp;
4017 
4018         ASSERT(dbp->db_mblk == mp);
4019         ASSERT(qindex < esbq_nelem);
4020 
4021         eqp = system_esbq_array;
4022         if (eqp != NULL) {
4023                 eqp += qindex;
4024         } else {
4025                 mutex_enter(&esbq_lock);
4026                 if (kmem_ready && system_esbq_array == NULL)
4027                         system_esbq_array = (esb_queue_t *)kmem_zalloc(
4028                             esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
4029                 mutex_exit(&esbq_lock);
4030                 eqp = system_esbq_array;
4031                 if (eqp != NULL)
4032                         eqp += qindex;
4033                 else
4034                         eqp = &system_esbq;
4035         }
4036 
4037         /*
4038          * Check data sanity. The dblock should have non-empty free function.
4039          * It is better to panic here then later when the dblock is freed
4040          * asynchronously when the context is lost.
4041          */
4042         if (dbp->db_frtnp->free_func == NULL) {
4043                 panic("freebs_enqueue: dblock %p has a NULL free callback",
4044                     (void *)dbp);
4045         }
4046 
4047         mutex_enter(&eqp->eq_lock);
4048         /* queue the new mblk on the esballoc queue */
4049         if (eqp->eq_head == NULL) {
4050                 eqp->eq_head = eqp->eq_tail = mp;
4051         } else {
4052                 eqp->eq_tail->b_next = mp;
4053                 eqp->eq_tail = mp;
4054         }
4055         eqp->eq_len++;
4056 
4057         /* If we're the first thread to reach the threshold, process */
4058         if (eqp->eq_len >= esbq_max_qlen &&
4059             !(eqp->eq_flags & ESBQ_PROCESSING))
4060                 esballoc_process_queue(eqp);
4061 
4062         esballoc_set_timer(eqp, esbq_timeout);
4063         mutex_exit(&eqp->eq_lock);
4064 }
4065 
4066 static void
4067 esballoc_process_queue(esb_queue_t *eqp)
4068 {
4069         mblk_t  *mp;
4070 
4071         ASSERT(MUTEX_HELD(&eqp->eq_lock));
4072 
4073         eqp->eq_flags |= ESBQ_PROCESSING;
4074 
4075         do {
4076                 /*
4077                  * Detach the message chain for processing.
4078                  */
4079                 mp = eqp->eq_head;
4080                 eqp->eq_tail->b_next = NULL;
4081                 eqp->eq_head = eqp->eq_tail = NULL;
4082                 eqp->eq_len = 0;
4083                 mutex_exit(&eqp->eq_lock);
4084 
4085                 /*
4086                  * Process the message chain.
4087                  */
4088                 esballoc_enqueue_mblk(mp);
4089                 mutex_enter(&eqp->eq_lock);
4090         } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
4091 
4092         eqp->eq_flags &= ~ESBQ_PROCESSING;
4093 }
4094 
4095 /*
4096  * taskq callback routine to free esballoced mblk's
4097  */
4098 static void
4099 esballoc_mblk_free(mblk_t *mp)
4100 {
4101         mblk_t  *nextmp;
4102 
4103         for (; mp != NULL; mp = nextmp) {
4104                 nextmp = mp->b_next;
4105                 mp->b_next = NULL;
4106                 mblk_free(mp);
4107         }
4108 }
4109 
4110 static void
4111 esballoc_enqueue_mblk(mblk_t *mp)
4112 {
4113 
4114         if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4115             TQ_NOSLEEP) == NULL) {
4116                 mblk_t *first_mp = mp;
4117                 /*
4118                  * System is low on resources and can't perform a non-sleeping
4119                  * dispatch. Schedule for a background thread.
4120                  */
4121                 mutex_enter(&service_queue);
4122                 STRSTAT(taskqfails);
4123 
4124                 while (mp->b_next != NULL)
4125                         mp = mp->b_next;
4126 
4127                 mp->b_next = freebs_list;
4128                 freebs_list = first_mp;
4129                 cv_signal(&services_to_run);
4130                 mutex_exit(&service_queue);
4131         }
4132 }
4133 
4134 static void
4135 esballoc_timer(void *arg)
4136 {
4137         esb_queue_t *eqp = arg;
4138 
4139         mutex_enter(&eqp->eq_lock);
4140         eqp->eq_flags &= ~ESBQ_TIMER;
4141 
4142         if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4143             eqp->eq_len > 0)
4144                 esballoc_process_queue(eqp);
4145 
4146         esballoc_set_timer(eqp, esbq_timeout);
4147         mutex_exit(&eqp->eq_lock);
4148 }
4149 
4150 static void
4151 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4152 {
4153         ASSERT(MUTEX_HELD(&eqp->eq_lock));
4154 
4155         if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4156                 (void) timeout(esballoc_timer, eqp, eq_timeout);
4157                 eqp->eq_flags |= ESBQ_TIMER;
4158         }
4159 }
4160 
4161 /*
4162  * Setup esbq array length based upon NCPU scaled by CPUs per
4163  * queue. Use static system_esbq until kmem_ready and we can
4164  * create an array in freebs_enqueue().
4165  */
4166 void
4167 esballoc_queue_init(void)
4168 {
4169         esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4170         esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4171         esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4172         system_esbq.eq_len = 0;
4173         system_esbq.eq_head = system_esbq.eq_tail = NULL;
4174         system_esbq.eq_flags = 0;
4175 }
4176 
4177 /*
4178  * Set the QBACK or QB_BACK flag in the given queue for
4179  * the given priority band.
4180  */
4181 void
4182 setqback(queue_t *q, unsigned char pri)
4183 {
4184         int i;
4185         qband_t *qbp;
4186         qband_t **qbpp;
4187 
4188         ASSERT(MUTEX_HELD(QLOCK(q)));
4189         if (pri != 0) {
4190                 if (pri > q->q_nband) {
4191                         qbpp = &q->q_bandp;
4192                         while (*qbpp)
4193                                 qbpp = &(*qbpp)->qb_next;
4194                         while (pri > q->q_nband) {
4195                                 if ((*qbpp = allocband()) == NULL) {
4196                                         cmn_err(CE_WARN,
4197                                             "setqback: can't allocate qband\n");
4198                                         return;
4199                                 }
4200                                 (*qbpp)->qb_hiwat = q->q_hiwat;
4201                                 (*qbpp)->qb_lowat = q->q_lowat;
4202                                 q->q_nband++;
4203                                 qbpp = &(*qbpp)->qb_next;
4204                         }
4205                 }
4206                 qbp = q->q_bandp;
4207                 i = pri;
4208                 while (--i)
4209                         qbp = qbp->qb_next;
4210                 qbp->qb_flag |= QB_BACK;
4211         } else {
4212                 q->q_flag |= QBACK;
4213         }
4214 }
4215 
4216 int
4217 strcopyin(void *from, void *to, size_t len, int copyflag)
4218 {
4219         if (copyflag & U_TO_K) {
4220                 ASSERT((copyflag & K_TO_K) == 0);
4221                 if (copyin(from, to, len))
4222                         return (EFAULT);
4223         } else {
4224                 ASSERT(copyflag & K_TO_K);
4225                 bcopy(from, to, len);
4226         }
4227         return (0);
4228 }
4229 
4230 int
4231 strcopyout(void *from, void *to, size_t len, int copyflag)
4232 {
4233         if (copyflag & U_TO_K) {
4234                 if (copyout(from, to, len))
4235                         return (EFAULT);
4236         } else {
4237                 ASSERT(copyflag & K_TO_K);
4238                 bcopy(from, to, len);
4239         }
4240         return (0);
4241 }
4242 
4243 /*
4244  * strsignal_nolock() posts a signal to the process(es) at the stream head.
4245  * It assumes that the stream head lock is already held, whereas strsignal()
4246  * acquires the lock first.  This routine was created because a few callers
4247  * release the stream head lock before calling only to re-acquire it after
4248  * it returns.
4249  */
4250 void
4251 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4252 {
4253         ASSERT(MUTEX_HELD(&stp->sd_lock));
4254         switch (sig) {
4255         case SIGPOLL:
4256                 if (stp->sd_sigflags & S_MSG)
4257                         strsendsig(stp->sd_siglist, S_MSG, band, 0);
4258                 break;
4259         default:
4260                 if (stp->sd_pgidp)
4261                         pgsignal(stp->sd_pgidp, sig);
4262                 break;
4263         }
4264 }
4265 
4266 void
4267 strsignal(stdata_t *stp, int sig, int32_t band)
4268 {
4269         TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4270             "strsignal:%p, %X, %X", stp, sig, band);
4271 
4272         mutex_enter(&stp->sd_lock);
4273         switch (sig) {
4274         case SIGPOLL:
4275                 if (stp->sd_sigflags & S_MSG)
4276                         strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4277                 break;
4278 
4279         default:
4280                 if (stp->sd_pgidp) {
4281                         pgsignal(stp->sd_pgidp, sig);
4282                 }
4283                 break;
4284         }
4285         mutex_exit(&stp->sd_lock);
4286 }
4287 
4288 void
4289 strhup(stdata_t *stp)
4290 {
4291         ASSERT(mutex_owned(&stp->sd_lock));
4292         pollwakeup(&stp->sd_pollist, POLLHUP);
4293         if (stp->sd_sigflags & S_HANGUP)
4294                 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4295 }
4296 
4297 /*
4298  * Backenable the first queue upstream from `q' with a service procedure.
4299  */
4300 void
4301 backenable(queue_t *q, uchar_t pri)
4302 {
4303         queue_t *nq;
4304 
4305         /*
4306          * Our presence might not prevent other modules in our own
4307          * stream from popping/pushing since the caller of getq might not
4308          * have a claim on the queue (some drivers do a getq on somebody
4309          * else's queue - they know that the queue itself is not going away
4310          * but the framework has to guarantee q_next in that stream).
4311          */
4312         claimstr(q);
4313 
4314         /* Find nearest back queue with service proc */
4315         for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4316                 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4317         }
4318 
4319         if (nq) {
4320                 kthread_t *freezer;
4321                 /*
4322                  * backenable can be called either with no locks held
4323                  * or with the stream frozen (the latter occurs when a module
4324                  * calls rmvq with the stream frozen). If the stream is frozen
4325                  * by the caller the caller will hold all qlocks in the stream.
4326                  * Note that a frozen stream doesn't freeze a mated stream,
4327                  * so we explicitly check for that.
4328                  */
4329                 freezer = STREAM(q)->sd_freezer;
4330                 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4331                         mutex_enter(QLOCK(nq));
4332                 }
4333 #ifdef DEBUG
4334                 else {
4335                         ASSERT(frozenstr(q));
4336                         ASSERT(MUTEX_HELD(QLOCK(q)));
4337                         ASSERT(MUTEX_HELD(QLOCK(nq)));
4338                 }
4339 #endif
4340                 setqback(nq, pri);
4341                 qenable_locked(nq);
4342                 if (freezer != curthread || STREAM(q) != STREAM(nq))
4343                         mutex_exit(QLOCK(nq));
4344         }
4345         releasestr(q);
4346 }
4347 
4348 /*
4349  * Return the appropriate errno when one of flags_to_check is set
4350  * in sd_flags. Uses the exported error routines if they are set.
4351  * Will return 0 if non error is set (or if the exported error routines
4352  * do not return an error).
4353  *
4354  * If there is both a read and write error to check, we prefer the read error.
4355  * Also, give preference to recorded errno's over the error functions.
4356  * The flags that are handled are:
4357  *      STPLEX          return EINVAL
4358  *      STRDERR         return sd_rerror (and clear if STRDERRNONPERSIST)
4359  *      STWRERR         return sd_werror (and clear if STWRERRNONPERSIST)
4360  *      STRHUP          return sd_werror
4361  *
4362  * If the caller indicates that the operation is a peek, a nonpersistent error
4363  * is not cleared.
4364  */
4365 int
4366 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4367 {
4368         int32_t sd_flag = stp->sd_flag & flags_to_check;
4369         int error = 0;
4370 
4371         ASSERT(MUTEX_HELD(&stp->sd_lock));
4372         ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4373         if (sd_flag & STPLEX)
4374                 error = EINVAL;
4375         else if (sd_flag & STRDERR) {
4376                 error = stp->sd_rerror;
4377                 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4378                         /*
4379                          * Read errors are non-persistent i.e. discarded once
4380                          * returned to a non-peeking caller,
4381                          */
4382                         stp->sd_rerror = 0;
4383                         stp->sd_flag &= ~STRDERR;
4384                 }
4385                 if (error == 0 && stp->sd_rderrfunc != NULL) {
4386                         int clearerr = 0;
4387 
4388                         error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4389                             &clearerr);
4390                         if (clearerr) {
4391                                 stp->sd_flag &= ~STRDERR;
4392                                 stp->sd_rderrfunc = NULL;
4393                         }
4394                 }
4395         } else if (sd_flag & STWRERR) {
4396                 error = stp->sd_werror;
4397                 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4398                         /*
4399                          * Write errors are non-persistent i.e. discarded once
4400                          * returned to a non-peeking caller,
4401                          */
4402                         stp->sd_werror = 0;
4403                         stp->sd_flag &= ~STWRERR;
4404                 }
4405                 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4406                         int clearerr = 0;
4407 
4408                         error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4409                             &clearerr);
4410                         if (clearerr) {
4411                                 stp->sd_flag &= ~STWRERR;
4412                                 stp->sd_wrerrfunc = NULL;
4413                         }
4414                 }
4415         } else if (sd_flag & STRHUP) {
4416                 /* sd_werror set when STRHUP */
4417                 error = stp->sd_werror;
4418         }
4419         return (error);
4420 }
4421 
4422 
4423 /*
4424  * Single-thread open/close/push/pop
4425  * for twisted streams also
4426  */
4427 int
4428 strstartplumb(stdata_t *stp, int flag, int cmd)
4429 {
4430         int waited = 1;
4431         int error = 0;
4432 
4433         if (STRMATED(stp)) {
4434                 struct stdata *stmatep = stp->sd_mate;
4435 
4436                 STRLOCKMATES(stp);
4437                 while (waited) {
4438                         waited = 0;
4439                         while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4440                                 if ((cmd == I_POP) &&
4441                                     (flag & (FNDELAY|FNONBLOCK))) {
4442                                         STRUNLOCKMATES(stp);
4443                                         return (EAGAIN);
4444                                 }
4445                                 waited = 1;
4446                                 mutex_exit(&stp->sd_lock);
4447                                 if (!cv_wait_sig(&stmatep->sd_monitor,
4448                                     &stmatep->sd_lock)) {
4449                                         mutex_exit(&stmatep->sd_lock);
4450                                         return (EINTR);
4451                                 }
4452                                 mutex_exit(&stmatep->sd_lock);
4453                                 STRLOCKMATES(stp);
4454                         }
4455                         while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4456                                 if ((cmd == I_POP) &&
4457                                     (flag & (FNDELAY|FNONBLOCK))) {
4458                                         STRUNLOCKMATES(stp);
4459                                         return (EAGAIN);
4460                                 }
4461                                 waited = 1;
4462                                 mutex_exit(&stmatep->sd_lock);
4463                                 if (!cv_wait_sig(&stp->sd_monitor,
4464                                     &stp->sd_lock)) {
4465                                         mutex_exit(&stp->sd_lock);
4466                                         return (EINTR);
4467                                 }
4468                                 mutex_exit(&stp->sd_lock);
4469                                 STRLOCKMATES(stp);
4470                         }
4471                         if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4472                                 error = strgeterr(stp,
4473                                     STRDERR|STWRERR|STRHUP|STPLEX, 0);
4474                                 if (error != 0) {
4475                                         STRUNLOCKMATES(stp);
4476                                         return (error);
4477                                 }
4478                         }
4479                 }
4480                 stp->sd_flag |= STRPLUMB;
4481                 STRUNLOCKMATES(stp);
4482         } else {
4483                 mutex_enter(&stp->sd_lock);
4484                 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4485                         if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4486                             (flag & (FNDELAY|FNONBLOCK))) {
4487                                 mutex_exit(&stp->sd_lock);
4488                                 return (EAGAIN);
4489                         }
4490                         if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4491                                 mutex_exit(&stp->sd_lock);
4492                                 return (EINTR);
4493                         }
4494                         if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4495                                 error = strgeterr(stp,
4496                                     STRDERR|STWRERR|STRHUP|STPLEX, 0);
4497                                 if (error != 0) {
4498                                         mutex_exit(&stp->sd_lock);
4499                                         return (error);
4500                                 }
4501                         }
4502                 }
4503                 stp->sd_flag |= STRPLUMB;
4504                 mutex_exit(&stp->sd_lock);
4505         }
4506         return (0);
4507 }
4508 
4509 /*
4510  * Complete the plumbing operation associated with stream `stp'.
4511  */
4512 void
4513 strendplumb(stdata_t *stp)
4514 {
4515         ASSERT(MUTEX_HELD(&stp->sd_lock));
4516         ASSERT(stp->sd_flag & STRPLUMB);
4517         stp->sd_flag &= ~STRPLUMB;
4518         cv_broadcast(&stp->sd_monitor);
4519 }
4520 
4521 /*
4522  * This describes how the STREAMS framework handles synchronization
4523  * during open/push and close/pop.
4524  * The key interfaces for open and close are qprocson and qprocsoff,
4525  * respectively. While the close case in general is harder both open
4526  * have close have significant similarities.
4527  *
4528  * During close the STREAMS framework has to both ensure that there
4529  * are no stale references to the queue pair (and syncq) that
4530  * are being closed and also provide the guarantees that are documented
4531  * in qprocsoff(9F).
4532  * If there are stale references to the queue that is closing it can
4533  * result in kernel memory corruption or kernel panics.
4534  *
4535  * Note that is it up to the module/driver to ensure that it itself
4536  * does not have any stale references to the closing queues once its close
4537  * routine returns. This includes:
4538  *  - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4539  *    associated with the queues. For timeout and bufcall callbacks the
4540  *    module/driver also has to ensure (or wait for) any callbacks that
4541  *    are in progress.
4542  *  - If the module/driver is using esballoc it has to ensure that any
4543  *    esballoc free functions do not refer to a queue that has closed.
4544  *    (Note that in general the close routine can not wait for the esballoc'ed
4545  *    messages to be freed since that can cause a deadlock.)
4546  *  - Cancelling any interrupts that refer to the closing queues and
4547  *    also ensuring that there are no interrupts in progress that will
4548  *    refer to the closing queues once the close routine returns.
4549  *  - For multiplexors removing any driver global state that refers to
4550  *    the closing queue and also ensuring that there are no threads in
4551  *    the multiplexor that has picked up a queue pointer but not yet
4552  *    finished using it.
4553  *
4554  * In addition, a driver/module can only reference the q_next pointer
4555  * in its open, close, put, or service procedures or in a
4556  * qtimeout/qbufcall callback procedure executing "on" the correct
4557  * stream. Thus it can not reference the q_next pointer in an interrupt
4558  * routine or a timeout, bufcall or esballoc callback routine. Likewise
4559  * it can not reference q_next of a different queue e.g. in a mux that
4560  * passes messages from one queues put/service procedure to another queue.
4561  * In all the cases when the driver/module can not access the q_next
4562  * field it must use the *next* versions e.g. canputnext instead of
4563  * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4564  *
4565  *
4566  * Assuming that the driver/module conforms to the above constraints
4567  * the STREAMS framework has to avoid stale references to q_next for all
4568  * the framework internal cases which include (but are not limited to):
4569  *  - Threads in canput/canputnext/backenable and elsewhere that are
4570  *    walking q_next.
4571  *  - Messages on a syncq that have a reference to the queue through b_queue.
4572  *  - Messages on an outer perimeter (syncq) that have a reference to the
4573  *    queue through b_queue.
4574  *  - Threads that use q_nfsrv (e.g. canput) to find a queue.
4575  *    Note that only canput and bcanput use q_nfsrv without any locking.
4576  *
4577  * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4578  * after qprocsoff returns, the framework has to ensure that no threads can
4579  * enter the put or service routines for the closing read or write-side queue.
4580  * In addition to preventing "direct" entry into the put procedures
4581  * the framework also has to prevent messages being drained from
4582  * the syncq or the outer perimeter.
4583  * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4584  * mechanism to prevent qwriter(PERIM_OUTER) from running after
4585  * qprocsoff has returned.
4586  * Note that if a module/driver uses put(9F) on one of its own queues
4587  * it is up to the module/driver to ensure that the put() doesn't
4588  * get called when the queue is closing.
4589  *
4590  *
4591  * The framework aspects of the above "contract" is implemented by
4592  * qprocsoff, removeq, and strlock:
4593  *  - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4594  *    entering the service procedures.
4595  *  - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4596  *    canputnext, backenable etc from dereferencing the q_next that will
4597  *    soon change.
4598  *  - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4599  *    or other q_next walker that uses claimstr/releasestr to finish.
4600  *  - optionally for every syncq in the stream strlock acquires all the
4601  *    sq_lock's and waits for all sq_counts to drop to a value that indicates
4602  *    that no thread executes in the put or service procedures and that no
4603  *    thread is draining into the module/driver. This ensures that no
4604  *    open, close, put, service, or qtimeout/qbufcall callback procedure is
4605  *    currently executing hence no such thread can end up with the old stale
4606  *    q_next value and no canput/backenable can have the old stale
4607  *    q_nfsrv/q_next.
4608  *  - qdetach (wait_svc) makes sure that any scheduled or running threads
4609  *    have either finished or observed the QWCLOSE flag and gone away.
4610  */
4611 
4612 
4613 /*
4614  * Get all the locks necessary to change q_next.
4615  *
4616  * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4617  * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4618  * the only threads inside the syncq are threads currently calling removeq().
4619  * Since threads calling removeq() are in the process of removing their queues
4620  * from the stream, we do not need to worry about them accessing a stale q_next
4621  * pointer and thus we do not need to wait for them to exit (in fact, waiting
4622  * for them can cause deadlock).
4623  *
4624  * This routine is subject to starvation since it does not set any flag to
4625  * prevent threads from entering a module in the stream (i.e. sq_count can
4626  * increase on some syncq while it is waiting on some other syncq).
4627  *
4628  * Assumes that only one thread attempts to call strlock for a given
4629  * stream. If this is not the case the two threads would deadlock.
4630  * This assumption is guaranteed since strlock is only called by insertq
4631  * and removeq and streams plumbing changes are single-threaded for
4632  * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4633  *
4634  * For pipes, it is not difficult to atomically designate a pair of streams
4635  * to be mated. Once mated atomically by the framework the twisted pair remain
4636  * configured that way until dismantled atomically by the framework.
4637  * When plumbing takes place on a twisted stream it is necessary to ensure that
4638  * this operation is done exclusively on the twisted stream since two such
4639  * operations, each initiated on different ends of the pipe will deadlock
4640  * waiting for each other to complete.
4641  *
4642  * On entry, no locks should be held.
4643  * The locks acquired and held by strlock depends on a few factors.
4644  * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4645  *   and held on exit and all sq_count are at an acceptable level.
4646  * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4647  *   sd_refcnt being zero.
4648  */
4649 
4650 static void
4651 strlock(struct stdata *stp, sqlist_t *sqlist)
4652 {
4653         syncql_t *sql, *sql2;
4654 retry:
4655         /*
4656          * Wait for any claimstr to go away.
4657          */
4658         if (STRMATED(stp)) {
4659                 struct stdata *stp1, *stp2;
4660 
4661                 STRLOCKMATES(stp);
4662                 /*
4663                  * Note that the selection of locking order is not
4664                  * important, just that they are always acquired in
4665                  * the same order.  To assure this, we choose this
4666                  * order based on the value of the pointer, and since
4667                  * the pointer will not change for the life of this
4668                  * pair, we will always grab the locks in the same
4669                  * order (and hence, prevent deadlocks).
4670                  */
4671                 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4672                         stp1 = stp;
4673                         stp2 = stp->sd_mate;
4674                 } else {
4675                         stp2 = stp;
4676                         stp1 = stp->sd_mate;
4677                 }
4678                 mutex_enter(&stp1->sd_reflock);
4679                 if (stp1->sd_refcnt > 0) {
4680                         STRUNLOCKMATES(stp);
4681                         cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4682                         mutex_exit(&stp1->sd_reflock);
4683                         goto retry;
4684                 }
4685                 mutex_enter(&stp2->sd_reflock);
4686                 if (stp2->sd_refcnt > 0) {
4687                         STRUNLOCKMATES(stp);
4688                         mutex_exit(&stp1->sd_reflock);
4689                         cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4690                         mutex_exit(&stp2->sd_reflock);
4691                         goto retry;
4692                 }
4693                 STREAM_PUTLOCKS_ENTER(stp1);
4694                 STREAM_PUTLOCKS_ENTER(stp2);
4695         } else {
4696                 mutex_enter(&stp->sd_lock);
4697                 mutex_enter(&stp->sd_reflock);
4698                 while (stp->sd_refcnt > 0) {
4699                         mutex_exit(&stp->sd_lock);
4700                         cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4701                         if (mutex_tryenter(&stp->sd_lock) == 0) {
4702                                 mutex_exit(&stp->sd_reflock);
4703                                 mutex_enter(&stp->sd_lock);
4704                                 mutex_enter(&stp->sd_reflock);
4705                         }
4706                 }
4707                 STREAM_PUTLOCKS_ENTER(stp);
4708         }
4709 
4710         if (sqlist == NULL)
4711                 return;
4712 
4713         for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4714                 syncq_t *sq = sql->sql_sq;
4715                 uint16_t count;
4716 
4717                 mutex_enter(SQLOCK(sq));
4718                 count = sq->sq_count;
4719                 ASSERT(sq->sq_rmqcount <= count);
4720                 SQ_PUTLOCKS_ENTER(sq);
4721                 SUM_SQ_PUTCOUNTS(sq, count);
4722                 if (count == sq->sq_rmqcount)
4723                         continue;
4724 
4725                 /* Failed - drop all locks that we have acquired so far */
4726                 if (STRMATED(stp)) {
4727                         STREAM_PUTLOCKS_EXIT(stp);
4728                         STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4729                         STRUNLOCKMATES(stp);
4730                         mutex_exit(&stp->sd_reflock);
4731                         mutex_exit(&stp->sd_mate->sd_reflock);
4732                 } else {
4733                         STREAM_PUTLOCKS_EXIT(stp);
4734                         mutex_exit(&stp->sd_lock);
4735                         mutex_exit(&stp->sd_reflock);
4736                 }
4737                 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4738                     sql2 = sql2->sql_next) {
4739                         SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4740                         mutex_exit(SQLOCK(sql2->sql_sq));
4741                 }
4742 
4743                 /*
4744                  * The wait loop below may starve when there are many threads
4745                  * claiming the syncq. This is especially a problem with permod
4746                  * syncqs (IP). To lessen the impact of the problem we increment
4747                  * sq_needexcl and clear fastbits so that putnexts will slow
4748                  * down and call sqenable instead of draining right away.
4749                  */
4750                 sq->sq_needexcl++;
4751                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4752                 while (count > sq->sq_rmqcount) {
4753                         sq->sq_flags |= SQ_WANTWAKEUP;
4754                         SQ_PUTLOCKS_EXIT(sq);
4755                         cv_wait(&sq->sq_wait, SQLOCK(sq));
4756                         count = sq->sq_count;
4757                         SQ_PUTLOCKS_ENTER(sq);
4758                         SUM_SQ_PUTCOUNTS(sq, count);
4759                 }
4760                 sq->sq_needexcl--;
4761                 if (sq->sq_needexcl == 0)
4762                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4763                 SQ_PUTLOCKS_EXIT(sq);
4764                 ASSERT(count == sq->sq_rmqcount);
4765                 mutex_exit(SQLOCK(sq));
4766                 goto retry;
4767         }
4768 }
4769 
4770 /*
4771  * Drop all the locks that strlock acquired.
4772  */
4773 static void
4774 strunlock(struct stdata *stp, sqlist_t *sqlist)
4775 {
4776         syncql_t *sql;
4777 
4778         if (STRMATED(stp)) {
4779                 STREAM_PUTLOCKS_EXIT(stp);
4780                 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4781                 STRUNLOCKMATES(stp);
4782                 mutex_exit(&stp->sd_reflock);
4783                 mutex_exit(&stp->sd_mate->sd_reflock);
4784         } else {
4785                 STREAM_PUTLOCKS_EXIT(stp);
4786                 mutex_exit(&stp->sd_lock);
4787                 mutex_exit(&stp->sd_reflock);
4788         }
4789 
4790         if (sqlist == NULL)
4791                 return;
4792 
4793         for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4794                 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4795                 mutex_exit(SQLOCK(sql->sql_sq));
4796         }
4797 }
4798 
4799 /*
4800  * When the module has service procedure, we need check if the next
4801  * module which has service procedure is in flow control to trigger
4802  * the backenable.
4803  */
4804 static void
4805 backenable_insertedq(queue_t *q)
4806 {
4807         qband_t *qbp;
4808 
4809         claimstr(q);
4810         if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4811                 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4812                         backenable(q, 0);
4813 
4814                 qbp = q->q_next->q_nfsrv->q_bandp;
4815                 for (; qbp != NULL; qbp = qbp->qb_next)
4816                         if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4817                                 backenable(q, qbp->qb_first->b_band);
4818         }
4819         releasestr(q);
4820 }
4821 
4822 /*
4823  * Given two read queues, insert a new single one after another.
4824  *
4825  * This routine acquires all the necessary locks in order to change
4826  * q_next and related pointer using strlock().
4827  * It depends on the stream head ensuring that there are no concurrent
4828  * insertq or removeq on the same stream. The stream head ensures this
4829  * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4830  *
4831  * Note that no syncq locks are held during the q_next change. This is
4832  * applied to all streams since, unlike removeq, there is no problem of stale
4833  * pointers when adding a module to the stream. Thus drivers/modules that do a
4834  * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4835  * applied this optimization to all streams.
4836  */
4837 void
4838 insertq(struct stdata *stp, queue_t *new)
4839 {
4840         queue_t *after;
4841         queue_t *wafter;
4842         queue_t *wnew = _WR(new);
4843         boolean_t have_fifo = B_FALSE;
4844 
4845         if (new->q_flag & _QINSERTING) {
4846                 ASSERT(stp->sd_vnode->v_type != VFIFO);
4847                 after = new->q_next;
4848                 wafter = _WR(new->q_next);
4849         } else {
4850                 after = _RD(stp->sd_wrq);
4851                 wafter = stp->sd_wrq;
4852         }
4853 
4854         TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4855             "insertq:%p, %p", after, new);
4856         ASSERT(after->q_flag & QREADR);
4857         ASSERT(new->q_flag & QREADR);
4858 
4859         strlock(stp, NULL);
4860 
4861         /* Do we have a FIFO? */
4862         if (wafter->q_next == after) {
4863                 have_fifo = B_TRUE;
4864                 wnew->q_next = new;
4865         } else {
4866                 wnew->q_next = wafter->q_next;
4867         }
4868         new->q_next = after;
4869 
4870         set_nfsrv_ptr(new, wnew, after, wafter);
4871         /*
4872          * set_nfsrv_ptr() needs to know if this is an insertion or not,
4873          * so only reset this flag after calling it.
4874          */
4875         new->q_flag &= ~_QINSERTING;
4876 
4877         if (have_fifo) {
4878                 wafter->q_next = wnew;
4879         } else {
4880                 if (wafter->q_next)
4881                         _OTHERQ(wafter->q_next)->q_next = new;
4882                 wafter->q_next = wnew;
4883         }
4884 
4885         set_qend(new);
4886         /* The QEND flag might have to be updated for the upstream guy */
4887         set_qend(after);
4888 
4889         ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4890         ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4891         ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4892         ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4893         strsetuio(stp);
4894 
4895         /*
4896          * If this was a module insertion, bump the push count.
4897          */
4898         if (!(new->q_flag & QISDRV))
4899                 stp->sd_pushcnt++;
4900 
4901         strunlock(stp, NULL);
4902 
4903         /* check if the write Q needs backenable */
4904         backenable_insertedq(wnew);
4905 
4906         /* check if the read Q needs backenable */
4907         backenable_insertedq(new);
4908 }
4909 
4910 /*
4911  * Given a read queue, unlink it from any neighbors.
4912  *
4913  * This routine acquires all the necessary locks in order to
4914  * change q_next and related pointers and also guard against
4915  * stale references (e.g. through q_next) to the queue that
4916  * is being removed. It also plays part of the role in ensuring
4917  * that the module's/driver's put procedure doesn't get called
4918  * after qprocsoff returns.
4919  *
4920  * Removeq depends on the stream head ensuring that there are
4921  * no concurrent insertq or removeq on the same stream. The
4922  * stream head ensures this using the flags STWOPEN, STRCLOSE and
4923  * STRPLUMB.
4924  *
4925  * The set of locks needed to remove the queue is different in
4926  * different cases:
4927  *
4928  * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4929  * waiting for the syncq reference count to drop to 0 indicating that no
4930  * non-close threads are present anywhere in the stream. This ensures that any
4931  * module/driver can reference q_next in its open, close, put, or service
4932  * procedures.
4933  *
4934  * The sq_rmqcount counter tracks the number of threads inside removeq().
4935  * strlock() ensures that there is either no threads executing inside perimeter
4936  * or there is only a thread calling qprocsoff().
4937  *
4938  * strlock() compares the value of sq_count with the number of threads inside
4939  * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4940  * any threads waiting in strlock() when the sq_rmqcount increases.
4941  */
4942 
4943 void
4944 removeq(queue_t *qp)
4945 {
4946         queue_t *wqp = _WR(qp);
4947         struct stdata *stp = STREAM(qp);
4948         sqlist_t *sqlist = NULL;
4949         boolean_t isdriver;
4950         int moved;
4951         syncq_t *sq = qp->q_syncq;
4952         syncq_t *wsq = wqp->q_syncq;
4953 
4954         ASSERT(stp);
4955 
4956         TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4957             "removeq:%p %p", qp, wqp);
4958         ASSERT(qp->q_flag&QREADR);
4959 
4960         /*
4961          * For queues using Synchronous streams, we must wait for all threads in
4962          * rwnext() to drain out before proceeding.
4963          */
4964         if (qp->q_flag & QSYNCSTR) {
4965                 /* First, we need wakeup any threads blocked in rwnext() */
4966                 mutex_enter(SQLOCK(sq));
4967                 if (sq->sq_flags & SQ_WANTWAKEUP) {
4968                         sq->sq_flags &= ~SQ_WANTWAKEUP;
4969                         cv_broadcast(&sq->sq_wait);
4970                 }
4971                 mutex_exit(SQLOCK(sq));
4972 
4973                 if (wsq != sq) {
4974                         mutex_enter(SQLOCK(wsq));
4975                         if (wsq->sq_flags & SQ_WANTWAKEUP) {
4976                                 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4977                                 cv_broadcast(&wsq->sq_wait);
4978                         }
4979                         mutex_exit(SQLOCK(wsq));
4980                 }
4981 
4982                 mutex_enter(QLOCK(qp));
4983                 while (qp->q_rwcnt > 0) {
4984                         qp->q_flag |= QWANTRMQSYNC;
4985                         cv_wait(&qp->q_wait, QLOCK(qp));
4986                 }
4987                 mutex_exit(QLOCK(qp));
4988 
4989                 mutex_enter(QLOCK(wqp));
4990                 while (wqp->q_rwcnt > 0) {
4991                         wqp->q_flag |= QWANTRMQSYNC;
4992                         cv_wait(&wqp->q_wait, QLOCK(wqp));
4993                 }
4994                 mutex_exit(QLOCK(wqp));
4995         }
4996 
4997         mutex_enter(SQLOCK(sq));
4998         sq->sq_rmqcount++;
4999         if (sq->sq_flags & SQ_WANTWAKEUP) {
5000                 sq->sq_flags &= ~SQ_WANTWAKEUP;
5001                 cv_broadcast(&sq->sq_wait);
5002         }
5003         mutex_exit(SQLOCK(sq));
5004 
5005         isdriver = (qp->q_flag & QISDRV);
5006 
5007         sqlist = sqlist_build(qp, stp, STRMATED(stp));
5008         strlock(stp, sqlist);
5009 
5010         reset_nfsrv_ptr(qp, wqp);
5011 
5012         ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
5013         ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
5014         /* Do we have a FIFO? */
5015         if (wqp->q_next == qp) {
5016                 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
5017         } else {
5018                 if (wqp->q_next)
5019                         backq(qp)->q_next = qp->q_next;
5020                 if (qp->q_next)
5021                         backq(wqp)->q_next = wqp->q_next;
5022         }
5023 
5024         /* The QEND flag might have to be updated for the upstream guy */
5025         if (qp->q_next)
5026                 set_qend(qp->q_next);
5027 
5028         ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
5029         ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
5030 
5031         /*
5032          * Move any messages destined for the put procedures to the next
5033          * syncq in line. Otherwise free them.
5034          */
5035         moved = 0;
5036         /*
5037          * Quick check to see whether there are any messages or events.
5038          */
5039         if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
5040                 moved += propagate_syncq(qp);
5041         if (wqp->q_syncqmsgs != 0 ||
5042             (wqp->q_syncq->sq_flags & SQ_EVENTS))
5043                 moved += propagate_syncq(wqp);
5044 
5045         strsetuio(stp);
5046 
5047         /*
5048          * If this was a module removal, decrement the push count.
5049          */
5050         if (!isdriver)
5051                 stp->sd_pushcnt--;
5052 
5053         strunlock(stp, sqlist);
5054         sqlist_free(sqlist);
5055 
5056         /*
5057          * Make sure any messages that were propagated are drained.
5058          * Also clear any QFULL bit caused by messages that were propagated.
5059          */
5060 
5061         if (qp->q_next != NULL) {
5062                 clr_qfull(qp);
5063                 /*
5064                  * For the driver calling qprocsoff, propagate_syncq
5065                  * frees all the messages instead of putting it in
5066                  * the stream head
5067                  */
5068                 if (!isdriver && (moved > 0))
5069                         emptysq(qp->q_next->q_syncq);
5070         }
5071         if (wqp->q_next != NULL) {
5072                 clr_qfull(wqp);
5073                 /*
5074                  * We come here for any pop of a module except for the
5075                  * case of driver being removed. We don't call emptysq
5076                  * if we did not move any messages. This will avoid holding
5077                  * PERMOD syncq locks in emptysq
5078                  */
5079                 if (moved > 0)
5080                         emptysq(wqp->q_next->q_syncq);
5081         }
5082 
5083         mutex_enter(SQLOCK(sq));
5084         sq->sq_rmqcount--;
5085         mutex_exit(SQLOCK(sq));
5086 }
5087 
5088 /*
5089  * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
5090  * SQ_WRITER) on a syncq.
5091  * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5092  * sync queue and waits until sq_count reaches maxcnt.
5093  *
5094  * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5095  * does not care about putnext threads that are in the middle of calling put
5096  * entry points.
5097  *
5098  * This routine is used for both inner and outer syncqs.
5099  */
5100 static void
5101 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5102 {
5103         uint16_t count = 0;
5104 
5105         mutex_enter(SQLOCK(sq));
5106         /*
5107          * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5108          * SQ_FROZEN will be set if there is a frozen stream that has a
5109          * queue which also refers to this "shared" syncq.
5110          * SQ_BLOCKED will be set if there is "off" queue which also
5111          * refers to this "shared" syncq.
5112          */
5113         if (maxcnt != -1) {
5114                 count = sq->sq_count;
5115                 SQ_PUTLOCKS_ENTER(sq);
5116                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5117                 SUM_SQ_PUTCOUNTS(sq, count);
5118         }
5119         sq->sq_needexcl++;
5120         ASSERT(sq->sq_needexcl != 0);        /* wraparound */
5121 
5122         while ((sq->sq_flags & flag) ||
5123             (maxcnt != -1 && count > (unsigned)maxcnt)) {
5124                 sq->sq_flags |= SQ_WANTWAKEUP;
5125                 if (maxcnt != -1) {
5126                         SQ_PUTLOCKS_EXIT(sq);
5127                 }
5128                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5129                 if (maxcnt != -1) {
5130                         count = sq->sq_count;
5131                         SQ_PUTLOCKS_ENTER(sq);
5132                         SUM_SQ_PUTCOUNTS(sq, count);
5133                 }
5134         }
5135         sq->sq_needexcl--;
5136         sq->sq_flags |= flag;
5137         ASSERT(maxcnt == -1 || count == maxcnt);
5138         if (maxcnt != -1) {
5139                 if (sq->sq_needexcl == 0) {
5140                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5141                 }
5142                 SQ_PUTLOCKS_EXIT(sq);
5143         } else if (sq->sq_needexcl == 0) {
5144                 SQ_PUTCOUNT_SETFAST(sq);
5145         }
5146 
5147         mutex_exit(SQLOCK(sq));
5148 }
5149 
5150 /*
5151  * Reset a flag that was set with blocksq.
5152  *
5153  * Can not use this routine to reset SQ_WRITER.
5154  *
5155  * If "isouter" is set then the syncq is assumed to be an outer perimeter
5156  * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5157  * to handle the queued qwriter operations.
5158  *
5159  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5160  * sq_putlocks are used.
5161  */
5162 static void
5163 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5164 {
5165         uint16_t flags;
5166 
5167         mutex_enter(SQLOCK(sq));
5168         ASSERT(resetflag != SQ_WRITER);
5169         ASSERT(sq->sq_flags & resetflag);
5170         flags = sq->sq_flags & ~resetflag;
5171         sq->sq_flags = flags;
5172         if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5173                 if (flags & SQ_WANTWAKEUP) {
5174                         flags &= ~SQ_WANTWAKEUP;
5175                         cv_broadcast(&sq->sq_wait);
5176                 }
5177                 sq->sq_flags = flags;
5178                 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5179                         if (!isouter) {
5180                                 /* drain_syncq drops SQLOCK */
5181                                 drain_syncq(sq);
5182                                 return;
5183                         }
5184                 }
5185         }
5186         mutex_exit(SQLOCK(sq));
5187 }
5188 
5189 /*
5190  * Reset a flag that was set with blocksq.
5191  * Does not drain the syncq. Use emptysq() for that.
5192  * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5193  *
5194  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5195  * sq_putlocks are used.
5196  */
5197 static int
5198 dropsq(syncq_t *sq, uint16_t resetflag)
5199 {
5200         uint16_t flags;
5201 
5202         mutex_enter(SQLOCK(sq));
5203         ASSERT(sq->sq_flags & resetflag);
5204         flags = sq->sq_flags & ~resetflag;
5205         if (flags & SQ_WANTWAKEUP) {
5206                 flags &= ~SQ_WANTWAKEUP;
5207                 cv_broadcast(&sq->sq_wait);
5208         }
5209         sq->sq_flags = flags;
5210         mutex_exit(SQLOCK(sq));
5211         if (flags & SQ_QUEUED)
5212                 return (1);
5213         return (0);
5214 }
5215 
5216 /*
5217  * Empty all the messages on a syncq.
5218  *
5219  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5220  * sq_putlocks are used.
5221  */
5222 static void
5223 emptysq(syncq_t *sq)
5224 {
5225         uint16_t flags;
5226 
5227         mutex_enter(SQLOCK(sq));
5228         flags = sq->sq_flags;
5229         if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5230                 /*
5231                  * To prevent potential recursive invocation of drain_syncq we
5232                  * do not call drain_syncq if count is non-zero.
5233                  */
5234                 if (sq->sq_count == 0) {
5235                         /* drain_syncq() drops SQLOCK */
5236                         drain_syncq(sq);
5237                         return;
5238                 } else
5239                         sqenable(sq);
5240         }
5241         mutex_exit(SQLOCK(sq));
5242 }
5243 
5244 /*
5245  * Ordered insert while removing duplicates.
5246  */
5247 static void
5248 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5249 {
5250         syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5251 
5252         prev_sqlpp = &sqlist->sqlist_head;
5253         while ((sqlp = *prev_sqlpp) != NULL) {
5254                 if (sqlp->sql_sq >= sqp) {
5255                         if (sqlp->sql_sq == sqp)     /* duplicate */
5256                                 return;
5257                         break;
5258                 }
5259                 prev_sqlpp = &sqlp->sql_next;
5260         }
5261         new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5262         ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5263         new_sqlp->sql_next = sqlp;
5264         new_sqlp->sql_sq = sqp;
5265         *prev_sqlpp = new_sqlp;
5266 }
5267 
5268 /*
5269  * Walk the write side queues until we hit either the driver
5270  * or a twist in the stream (_SAMESTR will return false in both
5271  * these cases) then turn around and walk the read side queues
5272  * back up to the stream head.
5273  */
5274 static void
5275 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5276 {
5277         while (q != NULL) {
5278                 sqlist_insert(sqlist, q->q_syncq);
5279 
5280                 if (_SAMESTR(q))
5281                         q = q->q_next;
5282                 else if (!(q->q_flag & QREADR))
5283                         q = _RD(q);
5284                 else
5285                         q = NULL;
5286         }
5287 }
5288 
5289 /*
5290  * Allocate and build a list of all syncqs in a stream and the syncq(s)
5291  * associated with the "q" parameter. The resulting list is sorted in a
5292  * canonical order and is free of duplicates.
5293  * Assumes the passed queue is a _RD(q).
5294  */
5295 static sqlist_t *
5296 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5297 {
5298         sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5299 
5300         /*
5301          * start with the current queue/qpair
5302          */
5303         ASSERT(q->q_flag & QREADR);
5304 
5305         sqlist_insert(sqlist, q->q_syncq);
5306         sqlist_insert(sqlist, _WR(q)->q_syncq);
5307 
5308         sqlist_insertall(sqlist, stp->sd_wrq);
5309         if (do_twist)
5310                 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5311 
5312         return (sqlist);
5313 }
5314 
5315 static sqlist_t *
5316 sqlist_alloc(struct stdata *stp, int kmflag)
5317 {
5318         size_t sqlist_size;
5319         sqlist_t *sqlist;
5320 
5321         /*
5322          * Allocate 2 syncql_t's for each pushed module. Note that
5323          * the sqlist_t structure already has 4 syncql_t's built in:
5324          * 2 for the stream head, and 2 for the driver/other stream head.
5325          */
5326         sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5327             sizeof (sqlist_t);
5328         if (STRMATED(stp))
5329                 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5330         sqlist = kmem_alloc(sqlist_size, kmflag);
5331 
5332         sqlist->sqlist_head = NULL;
5333         sqlist->sqlist_size = sqlist_size;
5334         sqlist->sqlist_index = 0;
5335 
5336         return (sqlist);
5337 }
5338 
5339 /*
5340  * Free the list created by sqlist_alloc()
5341  */
5342 static void
5343 sqlist_free(sqlist_t *sqlist)
5344 {
5345         kmem_free(sqlist, sqlist->sqlist_size);
5346 }
5347 
5348 /*
5349  * Prevent any new entries into any syncq in this stream.
5350  * Used by freezestr.
5351  */
5352 void
5353 strblock(queue_t *q)
5354 {
5355         struct stdata   *stp;
5356         syncql_t        *sql;
5357         sqlist_t        *sqlist;
5358 
5359         q = _RD(q);
5360 
5361         stp = STREAM(q);
5362         ASSERT(stp != NULL);
5363 
5364         /*
5365          * Get a sorted list with all the duplicates removed containing
5366          * all the syncqs referenced by this stream.
5367          */
5368         sqlist = sqlist_build(q, stp, B_FALSE);
5369         for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5370                 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5371         sqlist_free(sqlist);
5372 }
5373 
5374 /*
5375  * Release the block on new entries into this stream
5376  */
5377 void
5378 strunblock(queue_t *q)
5379 {
5380         struct stdata   *stp;
5381         syncql_t        *sql;
5382         sqlist_t        *sqlist;
5383         int             drain_needed;
5384 
5385         q = _RD(q);
5386 
5387         /*
5388          * Get a sorted list with all the duplicates removed containing
5389          * all the syncqs referenced by this stream.
5390          * Have to drop the SQ_FROZEN flag on all the syncqs before
5391          * starting to drain them; otherwise the draining might
5392          * cause a freezestr in some module on the stream (which
5393          * would deadlock).
5394          */
5395         stp = STREAM(q);
5396         ASSERT(stp != NULL);
5397         sqlist = sqlist_build(q, stp, B_FALSE);
5398         drain_needed = 0;
5399         for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5400                 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5401         if (drain_needed) {
5402                 for (sql = sqlist->sqlist_head; sql != NULL;
5403                     sql = sql->sql_next)
5404                         emptysq(sql->sql_sq);
5405         }
5406         sqlist_free(sqlist);
5407 }
5408 
5409 #ifdef DEBUG
5410 static int
5411 qprocsareon(queue_t *rq)
5412 {
5413         if (rq->q_next == NULL)
5414                 return (0);
5415         return (_WR(rq->q_next)->q_next == _WR(rq));
5416 }
5417 
5418 int
5419 qclaimed(queue_t *q)
5420 {
5421         uint_t count;
5422 
5423         count = q->q_syncq->sq_count;
5424         SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5425         return (count != 0);
5426 }
5427 
5428 /*
5429  * Check if anyone has frozen this stream with freezestr
5430  */
5431 int
5432 frozenstr(queue_t *q)
5433 {
5434         return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5435 }
5436 #endif /* DEBUG */
5437 
5438 /*
5439  * Enter a queue.
5440  * Obsoleted interface. Should not be used.
5441  */
5442 void
5443 enterq(queue_t *q)
5444 {
5445         entersq(q->q_syncq, SQ_CALLBACK);
5446 }
5447 
5448 void
5449 leaveq(queue_t *q)
5450 {
5451         leavesq(q->q_syncq, SQ_CALLBACK);
5452 }
5453 
5454 /*
5455  * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5456  * to check.
5457  * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5458  * calls and the running of open, close and service procedures.
5459  *
5460  * If c_inner bit is set no need to grab sq_putlocks since we don't care
5461  * if other threads have entered or are entering put entry point.
5462  *
5463  * If c_inner bit is set it might have been possible to use
5464  * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5465  * open/close path for IP) but since the count may need to be decremented in
5466  * qwait() we wouldn't know which counter to decrement. Currently counter is
5467  * selected by current cpu_seqid and current CPU can change at any moment. XXX
5468  * in the future we might use curthread id bits to select the counter and this
5469  * would stay constant across routine calls.
5470  */
5471 void
5472 entersq(syncq_t *sq, int entrypoint)
5473 {
5474         uint16_t        count = 0;
5475         uint16_t        flags;
5476         uint16_t        waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5477         uint16_t        type;
5478         uint_t          c_inner = entrypoint & SQ_CI;
5479         uint_t          c_outer = entrypoint & SQ_CO;
5480 
5481         /*
5482          * Increment ref count to keep closes out of this queue.
5483          */
5484         ASSERT(sq);
5485         ASSERT(c_inner && c_outer);
5486         mutex_enter(SQLOCK(sq));
5487         flags = sq->sq_flags;
5488         type = sq->sq_type;
5489         if (!(type & c_inner)) {
5490                 /* Make sure all putcounts now use slowlock. */
5491                 count = sq->sq_count;
5492                 SQ_PUTLOCKS_ENTER(sq);
5493                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5494                 SUM_SQ_PUTCOUNTS(sq, count);
5495                 sq->sq_needexcl++;
5496                 ASSERT(sq->sq_needexcl != 0);        /* wraparound */
5497                 waitflags |= SQ_MESSAGES;
5498         }
5499         /*
5500          * Wait until we can enter the inner perimeter.
5501          * If we want exclusive access we wait until sq_count is 0.
5502          * We have to do this before entering the outer perimeter in order
5503          * to preserve put/close message ordering.
5504          */
5505         while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5506                 sq->sq_flags = flags | SQ_WANTWAKEUP;
5507                 if (!(type & c_inner)) {
5508                         SQ_PUTLOCKS_EXIT(sq);
5509                 }
5510                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5511                 if (!(type & c_inner)) {
5512                         count = sq->sq_count;
5513                         SQ_PUTLOCKS_ENTER(sq);
5514                         SUM_SQ_PUTCOUNTS(sq, count);
5515                 }
5516                 flags = sq->sq_flags;
5517         }
5518 
5519         if (!(type & c_inner)) {
5520                 ASSERT(sq->sq_needexcl > 0);
5521                 sq->sq_needexcl--;
5522                 if (sq->sq_needexcl == 0) {
5523                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5524                 }
5525         }
5526 
5527         /* Check if we need to enter the outer perimeter */
5528         if (!(type & c_outer)) {
5529                 /*
5530                  * We have to enter the outer perimeter exclusively before
5531                  * we can increment sq_count to avoid deadlock. This implies
5532                  * that we have to re-check sq_flags and sq_count.
5533                  *
5534                  * is it possible to have c_inner set when c_outer is not set?
5535                  */
5536                 if (!(type & c_inner)) {
5537                         SQ_PUTLOCKS_EXIT(sq);
5538                 }
5539                 mutex_exit(SQLOCK(sq));
5540                 outer_enter(sq->sq_outer, SQ_GOAWAY);
5541                 mutex_enter(SQLOCK(sq));
5542                 flags = sq->sq_flags;
5543                 /*
5544                  * there should be no need to recheck sq_putcounts
5545                  * because outer_enter() has already waited for them to clear
5546                  * after setting SQ_WRITER.
5547                  */
5548                 count = sq->sq_count;
5549 #ifdef DEBUG
5550                 /*
5551                  * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5552                  * of doing an ASSERT internally. Others should do
5553                  * something like
5554                  *       ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5555                  * without the need to #ifdef DEBUG it.
5556                  */
5557                 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5558 #endif
5559                 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5560                     (!(type & c_inner) && count != 0)) {
5561                         sq->sq_flags = flags | SQ_WANTWAKEUP;
5562                         cv_wait(&sq->sq_wait, SQLOCK(sq));
5563                         count = sq->sq_count;
5564                         flags = sq->sq_flags;
5565                 }
5566         }
5567 
5568         sq->sq_count++;
5569         ASSERT(sq->sq_count != 0);   /* Wraparound */
5570         if (!(type & c_inner)) {
5571                 /* Exclusive entry */
5572                 ASSERT(sq->sq_count == 1);
5573                 sq->sq_flags |= SQ_EXCL;
5574                 if (type & c_outer) {
5575                         SQ_PUTLOCKS_EXIT(sq);
5576                 }
5577         }
5578         mutex_exit(SQLOCK(sq));
5579 }
5580 
5581 /*
5582  * Leave a syncq. Announce to framework that closes may proceed.
5583  * c_inner and c_outer specify which concurrency bits to check.
5584  *
5585  * Must never be called from driver or module put entry point.
5586  *
5587  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5588  * sq_putlocks are used.
5589  */
5590 void
5591 leavesq(syncq_t *sq, int entrypoint)
5592 {
5593         uint16_t        flags;
5594         uint16_t        type;
5595         uint_t          c_outer = entrypoint & SQ_CO;
5596 #ifdef DEBUG
5597         uint_t          c_inner = entrypoint & SQ_CI;
5598 #endif
5599 
5600         /*
5601          * Decrement ref count, drain the syncq if possible, and wake up
5602          * any waiting close.
5603          */
5604         ASSERT(sq);
5605         ASSERT(c_inner && c_outer);
5606         mutex_enter(SQLOCK(sq));
5607         flags = sq->sq_flags;
5608         type = sq->sq_type;
5609         if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5610 
5611                 if (flags & SQ_WANTWAKEUP) {
5612                         flags &= ~SQ_WANTWAKEUP;
5613                         cv_broadcast(&sq->sq_wait);
5614                 }
5615                 if (flags & SQ_WANTEXWAKEUP) {
5616                         flags &= ~SQ_WANTEXWAKEUP;
5617                         cv_broadcast(&sq->sq_exitwait);
5618                 }
5619 
5620                 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5621                         /*
5622                          * The syncq needs to be drained. "Exit" the syncq
5623                          * before calling drain_syncq.
5624                          */
5625                         ASSERT(sq->sq_count != 0);
5626                         sq->sq_count--;
5627                         ASSERT((flags & SQ_EXCL) || (type & c_inner));
5628                         sq->sq_flags = flags & ~SQ_EXCL;
5629                         drain_syncq(sq);
5630                         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5631                         /* Check if we need to exit the outer perimeter */
5632                         /* XXX will this ever be true? */
5633                         if (!(type & c_outer))
5634                                 outer_exit(sq->sq_outer);
5635                         return;
5636                 }
5637         }
5638         ASSERT(sq->sq_count != 0);
5639         sq->sq_count--;
5640         ASSERT((flags & SQ_EXCL) || (type & c_inner));
5641         sq->sq_flags = flags & ~SQ_EXCL;
5642         mutex_exit(SQLOCK(sq));
5643 
5644         /* Check if we need to exit the outer perimeter */
5645         if (!(sq->sq_type & c_outer))
5646                 outer_exit(sq->sq_outer);
5647 }
5648 
5649 /*
5650  * Prevent q_next from changing in this stream by incrementing sq_count.
5651  *
5652  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5653  * sq_putlocks are used.
5654  */
5655 void
5656 claimq(queue_t *qp)
5657 {
5658         syncq_t *sq = qp->q_syncq;
5659 
5660         mutex_enter(SQLOCK(sq));
5661         sq->sq_count++;
5662         ASSERT(sq->sq_count != 0);   /* Wraparound */
5663         mutex_exit(SQLOCK(sq));
5664 }
5665 
5666 /*
5667  * Undo claimq.
5668  *
5669  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5670  * sq_putlocks are used.
5671  */
5672 void
5673 releaseq(queue_t *qp)
5674 {
5675         syncq_t *sq = qp->q_syncq;
5676         uint16_t flags;
5677 
5678         mutex_enter(SQLOCK(sq));
5679         ASSERT(sq->sq_count > 0);
5680         sq->sq_count--;
5681 
5682         flags = sq->sq_flags;
5683         if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5684                 if (flags & SQ_WANTWAKEUP) {
5685                         flags &= ~SQ_WANTWAKEUP;
5686                         cv_broadcast(&sq->sq_wait);
5687                 }
5688                 sq->sq_flags = flags;
5689                 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5690                         /*
5691                          * To prevent potential recursive invocation of
5692                          * drain_syncq we do not call drain_syncq if count is
5693                          * non-zero.
5694                          */
5695                         if (sq->sq_count == 0) {
5696                                 drain_syncq(sq);
5697                                 return;
5698                         } else
5699                                 sqenable(sq);
5700                 }
5701         }
5702         mutex_exit(SQLOCK(sq));
5703 }
5704 
5705 /*
5706  * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5707  */
5708 void
5709 claimstr(queue_t *qp)
5710 {
5711         struct stdata *stp = STREAM(qp);
5712 
5713         mutex_enter(&stp->sd_reflock);
5714         stp->sd_refcnt++;
5715         ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5716         mutex_exit(&stp->sd_reflock);
5717 }
5718 
5719 /*
5720  * Undo claimstr.
5721  */
5722 void
5723 releasestr(queue_t *qp)
5724 {
5725         struct stdata *stp = STREAM(qp);
5726 
5727         mutex_enter(&stp->sd_reflock);
5728         ASSERT(stp->sd_refcnt != 0);
5729         if (--stp->sd_refcnt == 0)
5730                 cv_broadcast(&stp->sd_refmonitor);
5731         mutex_exit(&stp->sd_reflock);
5732 }
5733 
5734 static syncq_t *
5735 new_syncq(void)
5736 {
5737         return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5738 }
5739 
5740 static void
5741 free_syncq(syncq_t *sq)
5742 {
5743         ASSERT(sq->sq_head == NULL);
5744         ASSERT(sq->sq_outer == NULL);
5745         ASSERT(sq->sq_callbpend == NULL);
5746         ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5747             (sq->sq_onext == sq && sq->sq_oprev == sq));
5748 
5749         if (sq->sq_ciputctrl != NULL) {
5750                 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5751                 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5752                     sq->sq_nciputctrl, 0);
5753                 ASSERT(ciputctrl_cache != NULL);
5754                 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5755         }
5756 
5757         sq->sq_tail = NULL;
5758         sq->sq_evhead = NULL;
5759         sq->sq_evtail = NULL;
5760         sq->sq_ciputctrl = NULL;
5761         sq->sq_nciputctrl = 0;
5762         sq->sq_count = 0;
5763         sq->sq_rmqcount = 0;
5764         sq->sq_callbflags = 0;
5765         sq->sq_cancelid = 0;
5766         sq->sq_next = NULL;
5767         sq->sq_needexcl = 0;
5768         sq->sq_svcflags = 0;
5769         sq->sq_nqueues = 0;
5770         sq->sq_pri = 0;
5771         sq->sq_onext = NULL;
5772         sq->sq_oprev = NULL;
5773         sq->sq_flags = 0;
5774         sq->sq_type = 0;
5775         sq->sq_servcount = 0;
5776 
5777         kmem_cache_free(syncq_cache, sq);
5778 }
5779 
5780 /* Outer perimeter code */
5781 
5782 /*
5783  * The outer syncq uses the fields and flags in the syncq slightly
5784  * differently from the inner syncqs.
5785  *      sq_count        Incremented when there are pending or running
5786  *                      writers at the outer perimeter to prevent the set of
5787  *                      inner syncqs that belong to the outer perimeter from
5788  *                      changing.
5789  *      sq_head/tail    List of deferred qwriter(OUTER) operations.
5790  *
5791  *      SQ_BLOCKED      Set to prevent traversing of sq_next,sq_prev while
5792  *                      inner syncqs are added to or removed from the
5793  *                      outer perimeter.
5794  *      SQ_QUEUED       sq_head/tail has messages or events queued.
5795  *
5796  *      SQ_WRITER       A thread is currently traversing all the inner syncqs
5797  *                      setting the SQ_WRITER flag.
5798  */
5799 
5800 /*
5801  * Get write access at the outer perimeter.
5802  * Note that read access is done by entersq, putnext, and put by simply
5803  * incrementing sq_count in the inner syncq.
5804  *
5805  * Waits until "flags" is no longer set in the outer to prevent multiple
5806  * threads from having write access at the same time. SQ_WRITER has to be part
5807  * of "flags".
5808  *
5809  * Increases sq_count on the outer syncq to keep away outer_insert/remove
5810  * until the outer_exit is finished.
5811  *
5812  * outer_enter is vulnerable to starvation since it does not prevent new
5813  * threads from entering the inner syncqs while it is waiting for sq_count to
5814  * go to zero.
5815  */
5816 void
5817 outer_enter(syncq_t *outer, uint16_t flags)
5818 {
5819         syncq_t *sq;
5820         int     wait_needed;
5821         uint16_t        count;
5822 
5823         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5824             outer->sq_oprev != NULL);
5825         ASSERT(flags & SQ_WRITER);
5826 
5827 retry:
5828         mutex_enter(SQLOCK(outer));
5829         while (outer->sq_flags & flags) {
5830                 outer->sq_flags |= SQ_WANTWAKEUP;
5831                 cv_wait(&outer->sq_wait, SQLOCK(outer));
5832         }
5833 
5834         ASSERT(!(outer->sq_flags & SQ_WRITER));
5835         outer->sq_flags |= SQ_WRITER;
5836         outer->sq_count++;
5837         ASSERT(outer->sq_count != 0);        /* wraparound */
5838         wait_needed = 0;
5839         /*
5840          * Set SQ_WRITER on all the inner syncqs while holding
5841          * the SQLOCK on the outer syncq. This ensures that the changing
5842          * of SQ_WRITER is atomic under the outer SQLOCK.
5843          */
5844         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5845                 mutex_enter(SQLOCK(sq));
5846                 count = sq->sq_count;
5847                 SQ_PUTLOCKS_ENTER(sq);
5848                 sq->sq_flags |= SQ_WRITER;
5849                 SUM_SQ_PUTCOUNTS(sq, count);
5850                 if (count != 0)
5851                         wait_needed = 1;
5852                 SQ_PUTLOCKS_EXIT(sq);
5853                 mutex_exit(SQLOCK(sq));
5854         }
5855         mutex_exit(SQLOCK(outer));
5856 
5857         /*
5858          * Get everybody out of the syncqs sequentially.
5859          * Note that we don't actually need to acquire the PUTLOCKS, since
5860          * we have already cleared the fastbit, and set QWRITER.  By
5861          * definition, the count can not increase since putnext will
5862          * take the slowlock path (and the purpose of acquiring the
5863          * putlocks was to make sure it didn't increase while we were
5864          * waiting).
5865          *
5866          * Note that we still acquire the PUTLOCKS to be safe.
5867          */
5868         if (wait_needed) {
5869                 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5870                         mutex_enter(SQLOCK(sq));
5871                         count = sq->sq_count;
5872                         SQ_PUTLOCKS_ENTER(sq);
5873                         SUM_SQ_PUTCOUNTS(sq, count);
5874                         while (count != 0) {
5875                                 sq->sq_flags |= SQ_WANTWAKEUP;
5876                                 SQ_PUTLOCKS_EXIT(sq);
5877                                 cv_wait(&sq->sq_wait, SQLOCK(sq));
5878                                 count = sq->sq_count;
5879                                 SQ_PUTLOCKS_ENTER(sq);
5880                                 SUM_SQ_PUTCOUNTS(sq, count);
5881                         }
5882                         SQ_PUTLOCKS_EXIT(sq);
5883                         mutex_exit(SQLOCK(sq));
5884                 }
5885                 /*
5886                  * Verify that none of the flags got set while we
5887                  * were waiting for the sq_counts to drop.
5888                  * If this happens we exit and retry entering the
5889                  * outer perimeter.
5890                  */
5891                 mutex_enter(SQLOCK(outer));
5892                 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5893                         mutex_exit(SQLOCK(outer));
5894                         outer_exit(outer);
5895                         goto retry;
5896                 }
5897                 mutex_exit(SQLOCK(outer));
5898         }
5899 }
5900 
5901 /*
5902  * Drop the write access at the outer perimeter.
5903  * Read access is dropped implicitly (by putnext, put, and leavesq) by
5904  * decrementing sq_count.
5905  */
5906 void
5907 outer_exit(syncq_t *outer)
5908 {
5909         syncq_t *sq;
5910         int      drain_needed;
5911         uint16_t flags;
5912 
5913         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5914             outer->sq_oprev != NULL);
5915         ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5916 
5917         /*
5918          * Atomically (from the perspective of threads calling become_writer)
5919          * drop the write access at the outer perimeter by holding
5920          * SQLOCK(outer) across all the dropsq calls and the resetting of
5921          * SQ_WRITER.
5922          * This defines a locking order between the outer perimeter
5923          * SQLOCK and the inner perimeter SQLOCKs.
5924          */
5925         mutex_enter(SQLOCK(outer));
5926         flags = outer->sq_flags;
5927         ASSERT(outer->sq_flags & SQ_WRITER);
5928         if (flags & SQ_QUEUED) {
5929                 write_now(outer);
5930                 flags = outer->sq_flags;
5931         }
5932 
5933         /*
5934          * sq_onext is stable since sq_count has not yet been decreased.
5935          * Reset the SQ_WRITER flags in all syncqs.
5936          * After dropping SQ_WRITER on the outer syncq we empty all the
5937          * inner syncqs.
5938          */
5939         drain_needed = 0;
5940         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5941                 drain_needed += dropsq(sq, SQ_WRITER);
5942         ASSERT(!(outer->sq_flags & SQ_QUEUED));
5943         flags &= ~SQ_WRITER;
5944         if (drain_needed) {
5945                 outer->sq_flags = flags;
5946                 mutex_exit(SQLOCK(outer));
5947                 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5948                         emptysq(sq);
5949                 mutex_enter(SQLOCK(outer));
5950                 flags = outer->sq_flags;
5951         }
5952         if (flags & SQ_WANTWAKEUP) {
5953                 flags &= ~SQ_WANTWAKEUP;
5954                 cv_broadcast(&outer->sq_wait);
5955         }
5956         outer->sq_flags = flags;
5957         ASSERT(outer->sq_count > 0);
5958         outer->sq_count--;
5959         mutex_exit(SQLOCK(outer));
5960 }
5961 
5962 /*
5963  * Add another syncq to an outer perimeter.
5964  * Block out all other access to the outer perimeter while it is being
5965  * changed using blocksq.
5966  * Assumes that the caller has *not* done an outer_enter.
5967  *
5968  * Vulnerable to starvation in blocksq.
5969  */
5970 static void
5971 outer_insert(syncq_t *outer, syncq_t *sq)
5972 {
5973         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5974             outer->sq_oprev != NULL);
5975         ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5976             sq->sq_oprev == NULL);   /* Can't be in an outer perimeter */
5977 
5978         /* Get exclusive access to the outer perimeter list */
5979         blocksq(outer, SQ_BLOCKED, 0);
5980         ASSERT(outer->sq_flags & SQ_BLOCKED);
5981         ASSERT(!(outer->sq_flags & SQ_WRITER));
5982 
5983         mutex_enter(SQLOCK(sq));
5984         sq->sq_outer = outer;
5985         outer->sq_onext->sq_oprev = sq;
5986         sq->sq_onext = outer->sq_onext;
5987         outer->sq_onext = sq;
5988         sq->sq_oprev = outer;
5989         mutex_exit(SQLOCK(sq));
5990         unblocksq(outer, SQ_BLOCKED, 1);
5991 }
5992 
5993 /*
5994  * Remove a syncq from an outer perimeter.
5995  * Block out all other access to the outer perimeter while it is being
5996  * changed using blocksq.
5997  * Assumes that the caller has *not* done an outer_enter.
5998  *
5999  * Vulnerable to starvation in blocksq.
6000  */
6001 static void
6002 outer_remove(syncq_t *outer, syncq_t *sq)
6003 {
6004         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6005             outer->sq_oprev != NULL);
6006         ASSERT(sq->sq_outer == outer);
6007 
6008         /* Get exclusive access to the outer perimeter list */
6009         blocksq(outer, SQ_BLOCKED, 0);
6010         ASSERT(outer->sq_flags & SQ_BLOCKED);
6011         ASSERT(!(outer->sq_flags & SQ_WRITER));
6012 
6013         mutex_enter(SQLOCK(sq));
6014         sq->sq_outer = NULL;
6015         sq->sq_onext->sq_oprev = sq->sq_oprev;
6016         sq->sq_oprev->sq_onext = sq->sq_onext;
6017         sq->sq_oprev = sq->sq_onext = NULL;
6018         mutex_exit(SQLOCK(sq));
6019         unblocksq(outer, SQ_BLOCKED, 1);
6020 }
6021 
6022 /*
6023  * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
6024  * If this is the first callback for this outer perimeter then add
6025  * this outer perimeter to the list of outer perimeters that
6026  * the qwriter_outer_thread will process.
6027  *
6028  * Increments sq_count in the outer syncq to prevent the membership
6029  * of the outer perimeter (in terms of inner syncqs) to change while
6030  * the callback is pending.
6031  */
6032 static void
6033 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
6034 {
6035         ASSERT(MUTEX_HELD(SQLOCK(outer)));
6036 
6037         mp->b_prev = (mblk_t *)func;
6038         mp->b_queue = q;
6039         mp->b_next = NULL;
6040         outer->sq_count++;   /* Decremented when dequeued */
6041         ASSERT(outer->sq_count != 0);        /* Wraparound */
6042         if (outer->sq_evhead == NULL) {
6043                 /* First message. */
6044                 outer->sq_evhead = outer->sq_evtail = mp;
6045                 outer->sq_flags |= SQ_EVENTS;
6046                 mutex_exit(SQLOCK(outer));
6047                 STRSTAT(qwr_outer);
6048                 (void) taskq_dispatch(streams_taskq,
6049                     (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
6050         } else {
6051                 ASSERT(outer->sq_flags & SQ_EVENTS);
6052                 outer->sq_evtail->b_next = mp;
6053                 outer->sq_evtail = mp;
6054                 mutex_exit(SQLOCK(outer));
6055         }
6056 }
6057 
6058 /*
6059  * Try and upgrade to write access at the outer perimeter. If this can
6060  * not be done without blocking then queue the callback to be done
6061  * by the qwriter_outer_thread.
6062  *
6063  * This routine can only be called from put or service procedures plus
6064  * asynchronous callback routines that have properly entered the queue (with
6065  * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
6066  * associated with q.
6067  */
6068 void
6069 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
6070 {
6071         syncq_t *osq, *sq, *outer;
6072         int     failed;
6073         uint16_t flags;
6074 
6075         osq = q->q_syncq;
6076         outer = osq->sq_outer;
6077         if (outer == NULL)
6078                 panic("qwriter(PERIM_OUTER): no outer perimeter");
6079         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6080             outer->sq_oprev != NULL);
6081 
6082         mutex_enter(SQLOCK(outer));
6083         flags = outer->sq_flags;
6084         /*
6085          * If some thread is traversing sq_next, or if we are blocked by
6086          * outer_insert or outer_remove, or if the we already have queued
6087          * callbacks, then queue this callback for later processing.
6088          *
6089          * Also queue the qwriter for an interrupt thread in order
6090          * to reduce the time spent running at high IPL.
6091          * to identify there are events.
6092          */
6093         if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6094                 /*
6095                  * Queue the become_writer request.
6096                  * The queueing is atomic under SQLOCK(outer) in order
6097                  * to synchronize with outer_exit.
6098                  * queue_writer will drop the outer SQLOCK
6099                  */
6100                 if (flags & SQ_BLOCKED) {
6101                         /* Must set SQ_WRITER on inner perimeter */
6102                         mutex_enter(SQLOCK(osq));
6103                         osq->sq_flags |= SQ_WRITER;
6104                         mutex_exit(SQLOCK(osq));
6105                 } else {
6106                         if (!(flags & SQ_WRITER)) {
6107                                 /*
6108                                  * The outer could have been SQ_BLOCKED thus
6109                                  * SQ_WRITER might not be set on the inner.
6110                                  */
6111                                 mutex_enter(SQLOCK(osq));
6112                                 osq->sq_flags |= SQ_WRITER;
6113                                 mutex_exit(SQLOCK(osq));
6114                         }
6115                         ASSERT(osq->sq_flags & SQ_WRITER);
6116                 }
6117                 queue_writer(outer, func, q, mp);
6118                 return;
6119         }
6120         /*
6121          * We are half-way to exclusive access to the outer perimeter.
6122          * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6123          * while the inner syncqs are traversed.
6124          */
6125         outer->sq_count++;
6126         ASSERT(outer->sq_count != 0);        /* wraparound */
6127         flags |= SQ_WRITER;
6128         /*
6129          * Check if we can run the function immediately. Mark all
6130          * syncqs with the writer flag to prevent new entries into
6131          * put and service procedures.
6132          *
6133          * Set SQ_WRITER on all the inner syncqs while holding
6134          * the SQLOCK on the outer syncq. This ensures that the changing
6135          * of SQ_WRITER is atomic under the outer SQLOCK.
6136          */
6137         failed = 0;
6138         for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6139                 uint16_t count;
6140                 uint_t  maxcnt = (sq == osq) ? 1 : 0;
6141 
6142                 mutex_enter(SQLOCK(sq));
6143                 count = sq->sq_count;
6144                 SQ_PUTLOCKS_ENTER(sq);
6145                 SUM_SQ_PUTCOUNTS(sq, count);
6146                 if (sq->sq_count > maxcnt)
6147                         failed = 1;
6148                 sq->sq_flags |= SQ_WRITER;
6149                 SQ_PUTLOCKS_EXIT(sq);
6150                 mutex_exit(SQLOCK(sq));
6151         }
6152         if (failed) {
6153                 /*
6154                  * Some other thread has a read claim on the outer perimeter.
6155                  * Queue the callback for deferred processing.
6156                  *
6157                  * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6158                  * so that other qwriter(OUTER) calls will queue their
6159                  * callbacks as well. queue_writer increments sq_count so we
6160                  * decrement to compensate for the our increment.
6161                  *
6162                  * Dropping SQ_WRITER enables the writer thread to work
6163                  * on this outer perimeter.
6164                  */
6165                 outer->sq_flags = flags;
6166                 queue_writer(outer, func, q, mp);
6167                 /* queue_writer dropper the lock */
6168                 mutex_enter(SQLOCK(outer));
6169                 ASSERT(outer->sq_count > 0);
6170                 outer->sq_count--;
6171                 ASSERT(outer->sq_flags & SQ_WRITER);
6172                 flags = outer->sq_flags;
6173                 flags &= ~SQ_WRITER;
6174                 if (flags & SQ_WANTWAKEUP) {
6175                         flags &= ~SQ_WANTWAKEUP;
6176                         cv_broadcast(&outer->sq_wait);
6177                 }
6178                 outer->sq_flags = flags;
6179                 mutex_exit(SQLOCK(outer));
6180                 return;
6181         } else {
6182                 outer->sq_flags = flags;
6183                 mutex_exit(SQLOCK(outer));
6184         }
6185 
6186         /* Can run it immediately */
6187         (*func)(q, mp);
6188 
6189         outer_exit(outer);
6190 }
6191 
6192 /*
6193  * Dequeue all writer callbacks from the outer perimeter and run them.
6194  */
6195 static void
6196 write_now(syncq_t *outer)
6197 {
6198         mblk_t          *mp;
6199         queue_t         *q;
6200         void    (*func)();
6201 
6202         ASSERT(MUTEX_HELD(SQLOCK(outer)));
6203         ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6204             outer->sq_oprev != NULL);
6205         while ((mp = outer->sq_evhead) != NULL) {
6206                 /*
6207                  * queues cannot be placed on the queuelist on the outer
6208                  * perimeter.
6209                  */
6210                 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6211                 ASSERT((outer->sq_flags & SQ_EVENTS));
6212 
6213                 outer->sq_evhead = mp->b_next;
6214                 if (outer->sq_evhead == NULL) {
6215                         outer->sq_evtail = NULL;
6216                         outer->sq_flags &= ~SQ_EVENTS;
6217                 }
6218                 ASSERT(outer->sq_count != 0);
6219                 outer->sq_count--;   /* Incremented when enqueued. */
6220                 mutex_exit(SQLOCK(outer));
6221                 /*
6222                  * Drop the message if the queue is closing.
6223                  * Make sure that the queue is "claimed" when the callback
6224                  * is run in order to satisfy various ASSERTs.
6225                  */
6226                 q = mp->b_queue;
6227                 func = (void (*)())mp->b_prev;
6228                 ASSERT(func != NULL);
6229                 mp->b_next = mp->b_prev = NULL;
6230                 if (q->q_flag & QWCLOSE) {
6231                         freemsg(mp);
6232                 } else {
6233                         claimq(q);
6234                         (*func)(q, mp);
6235                         releaseq(q);
6236                 }
6237                 mutex_enter(SQLOCK(outer));
6238         }
6239         ASSERT(MUTEX_HELD(SQLOCK(outer)));
6240 }
6241 
6242 /*
6243  * The list of messages on the inner syncq is effectively hashed
6244  * by destination queue.  These destination queues are doubly
6245  * linked lists (hopefully) in priority order.  Messages are then
6246  * put on the queue referenced by the q_sqhead/q_sqtail elements.
6247  * Additional messages are linked together by the b_next/b_prev
6248  * elements in the mblk, with (similar to putq()) the first message
6249  * having a NULL b_prev and the last message having a NULL b_next.
6250  *
6251  * Events, such as qwriter callbacks, are put onto a list in FIFO
6252  * order referenced by sq_evhead, and sq_evtail.  This is a singly
6253  * linked list, and messages here MUST be processed in the order queued.
6254  */
6255 
6256 /*
6257  * Run the events on the syncq event list (sq_evhead).
6258  * Assumes there is only one claim on the syncq, it is
6259  * already exclusive (SQ_EXCL set), and the SQLOCK held.
6260  * Messages here are processed in order, with the SQ_EXCL bit
6261  * held all the way through till the last message is processed.
6262  */
6263 void
6264 sq_run_events(syncq_t *sq)
6265 {
6266         mblk_t          *bp;
6267         queue_t         *qp;
6268         uint16_t        flags = sq->sq_flags;
6269         void            (*func)();
6270 
6271         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6272         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6273             sq->sq_oprev == NULL) ||
6274             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6275             sq->sq_oprev != NULL));
6276 
6277         ASSERT(flags & SQ_EXCL);
6278         ASSERT(sq->sq_count == 1);
6279 
6280         /*
6281          * We need to process all of the events on this list.  It
6282          * is possible that new events will be added while we are
6283          * away processing a callback, so on every loop, we start
6284          * back at the beginning of the list.
6285          */
6286         /*
6287          * We have to reaccess sq_evhead since there is a
6288          * possibility of a new entry while we were running
6289          * the callback.
6290          */
6291         for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6292                 ASSERT(bp->b_queue->q_syncq == sq);
6293                 ASSERT(sq->sq_flags & SQ_EVENTS);
6294 
6295                 qp = bp->b_queue;
6296                 func = (void (*)())bp->b_prev;
6297                 ASSERT(func != NULL);
6298 
6299                 /*
6300                  * Messages from the event queue must be taken off in
6301                  * FIFO order.
6302                  */
6303                 ASSERT(sq->sq_evhead == bp);
6304                 sq->sq_evhead = bp->b_next;
6305 
6306                 if (bp->b_next == NULL) {
6307                         /* Deleting last */
6308                         ASSERT(sq->sq_evtail == bp);
6309                         sq->sq_evtail = NULL;
6310                         sq->sq_flags &= ~SQ_EVENTS;
6311                 }
6312                 bp->b_prev = bp->b_next = NULL;
6313                 ASSERT(bp->b_datap->db_ref != 0);
6314 
6315                 mutex_exit(SQLOCK(sq));
6316 
6317                 (*func)(qp, bp);
6318 
6319                 mutex_enter(SQLOCK(sq));
6320                 /*
6321                  * re-read the flags, since they could have changed.
6322                  */
6323                 flags = sq->sq_flags;
6324                 ASSERT(flags & SQ_EXCL);
6325         }
6326         ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6327         ASSERT(!(sq->sq_flags & SQ_EVENTS));
6328 
6329         if (flags & SQ_WANTWAKEUP) {
6330                 flags &= ~SQ_WANTWAKEUP;
6331                 cv_broadcast(&sq->sq_wait);
6332         }
6333         if (flags & SQ_WANTEXWAKEUP) {
6334                 flags &= ~SQ_WANTEXWAKEUP;
6335                 cv_broadcast(&sq->sq_exitwait);
6336         }
6337         sq->sq_flags = flags;
6338 }
6339 
6340 /*
6341  * Put messages on the event list.
6342  * If we can go exclusive now, do so and process the event list, otherwise
6343  * let the last claim service this list (or wake the sqthread).
6344  * This procedure assumes SQLOCK is held.  To run the event list, it
6345  * must be called with no claims.
6346  */
6347 static void
6348 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6349 {
6350         uint16_t count;
6351 
6352         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6353         ASSERT(func != NULL);
6354 
6355         /*
6356          * This is a callback.  Add it to the list of callbacks
6357          * and see about upgrading.
6358          */
6359         mp->b_prev = (mblk_t *)func;
6360         mp->b_queue = q;
6361         mp->b_next = NULL;
6362         if (sq->sq_evhead == NULL) {
6363                 sq->sq_evhead = sq->sq_evtail = mp;
6364                 sq->sq_flags |= SQ_EVENTS;
6365         } else {
6366                 ASSERT(sq->sq_evtail != NULL);
6367                 ASSERT(sq->sq_evtail->b_next == NULL);
6368                 ASSERT(sq->sq_flags & SQ_EVENTS);
6369                 sq->sq_evtail->b_next = mp;
6370                 sq->sq_evtail = mp;
6371         }
6372         /*
6373          * We have set SQ_EVENTS, so threads will have to
6374          * unwind out of the perimeter, and new entries will
6375          * not grab a putlock.  But we still need to know
6376          * how many threads have already made a claim to the
6377          * syncq, so grab the putlocks, and sum the counts.
6378          * If there are no claims on the syncq, we can upgrade
6379          * to exclusive, and run the event list.
6380          * NOTE: We hold the SQLOCK, so we can just grab the
6381          * putlocks.
6382          */
6383         count = sq->sq_count;
6384         SQ_PUTLOCKS_ENTER(sq);
6385         SUM_SQ_PUTCOUNTS(sq, count);
6386         /*
6387          * We have no claim, so we need to check if there
6388          * are no others, then we can upgrade.
6389          */
6390         /*
6391          * There are currently no claims on
6392          * the syncq by this thread (at least on this entry). The thread who has
6393          * the claim should drain syncq.
6394          */
6395         if (count > 0) {
6396                 /*
6397                  * Can't upgrade - other threads inside.
6398                  */
6399                 SQ_PUTLOCKS_EXIT(sq);
6400                 mutex_exit(SQLOCK(sq));
6401                 return;
6402         }
6403         /*
6404          * Need to set SQ_EXCL and make a claim on the syncq.
6405          */
6406         ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6407         sq->sq_flags |= SQ_EXCL;
6408         ASSERT(sq->sq_count == 0);
6409         sq->sq_count++;
6410         SQ_PUTLOCKS_EXIT(sq);
6411 
6412         /* Process the events list */
6413         sq_run_events(sq);
6414 
6415         /*
6416          * Release our claim...
6417          */
6418         sq->sq_count--;
6419 
6420         /*
6421          * And release SQ_EXCL.
6422          * We don't need to acquire the putlocks to release
6423          * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6424          */
6425         sq->sq_flags &= ~SQ_EXCL;
6426 
6427         /*
6428          * sq_run_events should have released SQ_EXCL
6429          */
6430         ASSERT(!(sq->sq_flags & SQ_EXCL));
6431 
6432         /*
6433          * If anything happened while we were running the
6434          * events (or was there before), we need to process
6435          * them now.  We shouldn't be exclusive sine we
6436          * released the perimeter above (plus, we asserted
6437          * for it).
6438          */
6439         if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6440                 drain_syncq(sq);
6441         else
6442                 mutex_exit(SQLOCK(sq));
6443 }
6444 
6445 /*
6446  * Perform delayed processing. The caller has to make sure that it is safe
6447  * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6448  * set).
6449  *
6450  * Assume that the caller has NO claims on the syncq.  However, a claim
6451  * on the syncq does not indicate that a thread is draining the syncq.
6452  * There may be more claims on the syncq than there are threads draining
6453  * (i.e.  #_threads_draining <= sq_count)
6454  *
6455  * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6456  * in order to preserve qwriter(OUTER) ordering constraints.
6457  *
6458  * sq_putcount only needs to be checked when dispatching the queued
6459  * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6460  */
6461 void
6462 drain_syncq(syncq_t *sq)
6463 {
6464         queue_t         *qp;
6465         uint16_t        count;
6466         uint16_t        type = sq->sq_type;
6467         uint16_t        flags = sq->sq_flags;
6468         boolean_t       bg_service = sq->sq_svcflags & SQ_SERVICE;
6469 
6470         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6471             "drain_syncq start:%p", sq);
6472         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6473         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6474             sq->sq_oprev == NULL) ||
6475             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6476             sq->sq_oprev != NULL));
6477 
6478         /*
6479          * Drop SQ_SERVICE flag.
6480          */
6481         if (bg_service)
6482                 sq->sq_svcflags &= ~SQ_SERVICE;
6483 
6484         /*
6485          * If SQ_EXCL is set, someone else is processing this syncq - let him
6486          * finish the job.
6487          */
6488         if (flags & SQ_EXCL) {
6489                 if (bg_service) {
6490                         ASSERT(sq->sq_servcount != 0);
6491                         sq->sq_servcount--;
6492                 }
6493                 mutex_exit(SQLOCK(sq));
6494                 return;
6495         }
6496 
6497         /*
6498          * This routine can be called by a background thread if
6499          * it was scheduled by a hi-priority thread.  SO, if there are
6500          * NOT messages queued, return (remember, we have the SQLOCK,
6501          * and it cannot change until we release it). Wakeup any waiters also.
6502          */
6503         if (!(flags & SQ_QUEUED)) {
6504                 if (flags & SQ_WANTWAKEUP) {
6505                         flags &= ~SQ_WANTWAKEUP;
6506                         cv_broadcast(&sq->sq_wait);
6507                 }
6508                 if (flags & SQ_WANTEXWAKEUP) {
6509                         flags &= ~SQ_WANTEXWAKEUP;
6510                         cv_broadcast(&sq->sq_exitwait);
6511                 }
6512                 sq->sq_flags = flags;
6513                 if (bg_service) {
6514                         ASSERT(sq->sq_servcount != 0);
6515                         sq->sq_servcount--;
6516                 }
6517                 mutex_exit(SQLOCK(sq));
6518                 return;
6519         }
6520 
6521         /*
6522          * If this is not a concurrent put perimeter, we need to
6523          * become exclusive to drain.  Also, if not CIPUT, we would
6524          * not have acquired a putlock, so we don't need to check
6525          * the putcounts.  If not entering with a claim, we test
6526          * for sq_count == 0.
6527          */
6528         type = sq->sq_type;
6529         if (!(type & SQ_CIPUT)) {
6530                 if (sq->sq_count > 1) {
6531                         if (bg_service) {
6532                                 ASSERT(sq->sq_servcount != 0);
6533                                 sq->sq_servcount--;
6534                         }
6535                         mutex_exit(SQLOCK(sq));
6536                         return;
6537                 }
6538                 sq->sq_flags |= SQ_EXCL;
6539         }
6540 
6541         /*
6542          * This is where we make a claim to the syncq.
6543          * This can either be done by incrementing a putlock, or
6544          * the sq_count.  But since we already have the SQLOCK
6545          * here, we just bump the sq_count.
6546          *
6547          * Note that after we make a claim, we need to let the code
6548          * fall through to the end of this routine to clean itself
6549          * up.  A return in the while loop will put the syncq in a
6550          * very bad state.
6551          */
6552         sq->sq_count++;
6553         ASSERT(sq->sq_count != 0);   /* wraparound */
6554 
6555         while ((flags = sq->sq_flags) & SQ_QUEUED) {
6556                 /*
6557                  * If we are told to stayaway or went exclusive,
6558                  * we are done.
6559                  */
6560                 if (flags & (SQ_STAYAWAY)) {
6561                         break;
6562                 }
6563 
6564                 /*
6565                  * If there are events to run, do so.
6566                  * We have one claim to the syncq, so if there are
6567                  * more than one, other threads are running.
6568                  */
6569                 if (sq->sq_evhead != NULL) {
6570                         ASSERT(sq->sq_flags & SQ_EVENTS);
6571 
6572                         count = sq->sq_count;
6573                         SQ_PUTLOCKS_ENTER(sq);
6574                         SUM_SQ_PUTCOUNTS(sq, count);
6575                         if (count > 1) {
6576                                 SQ_PUTLOCKS_EXIT(sq);
6577                                 /* Can't upgrade - other threads inside */
6578                                 break;
6579                         }
6580                         ASSERT((flags & SQ_EXCL) == 0);
6581                         sq->sq_flags = flags | SQ_EXCL;
6582                         SQ_PUTLOCKS_EXIT(sq);
6583                         /*
6584                          * we have the only claim, run the events,
6585                          * sq_run_events will clear the SQ_EXCL flag.
6586                          */
6587                         sq_run_events(sq);
6588 
6589                         /*
6590                          * If this is a CIPUT perimeter, we need
6591                          * to drop the SQ_EXCL flag so we can properly
6592                          * continue draining the syncq.
6593                          */
6594                         if (type & SQ_CIPUT) {
6595                                 ASSERT(sq->sq_flags & SQ_EXCL);
6596                                 sq->sq_flags &= ~SQ_EXCL;
6597                         }
6598 
6599                         /*
6600                          * And go back to the beginning just in case
6601                          * anything changed while we were away.
6602                          */
6603                         ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6604                         continue;
6605                 }
6606 
6607                 ASSERT(sq->sq_evhead == NULL);
6608                 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6609 
6610                 /*
6611                  * Find the queue that is not draining.
6612                  *
6613                  * q_draining is protected by QLOCK which we do not hold.
6614                  * But if it was set, then a thread was draining, and if it gets
6615                  * cleared, then it was because the thread has successfully
6616                  * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6617                  * state to happen, a thread needs the SQLOCK which we hold, and
6618                  * if there was such a flag, we would have already seen it.
6619                  */
6620 
6621                 for (qp = sq->sq_head;
6622                     qp != NULL && (qp->q_draining ||
6623                     (qp->q_sqflags & Q_SQDRAINING));
6624                     qp = qp->q_sqnext)
6625                         ;
6626 
6627                 if (qp == NULL)
6628                         break;
6629 
6630                 /*
6631                  * We have a queue to work on, and we hold the
6632                  * SQLOCK and one claim, call qdrain_syncq.
6633                  * This means we need to release the SQLOCK and
6634                  * acquire the QLOCK (OK since we have a claim).
6635                  * Note that qdrain_syncq will actually dequeue
6636                  * this queue from the sq_head list when it is
6637                  * convinced all the work is done and release
6638                  * the QLOCK before returning.
6639                  */
6640                 qp->q_sqflags |= Q_SQDRAINING;
6641                 mutex_exit(SQLOCK(sq));
6642                 mutex_enter(QLOCK(qp));
6643                 qdrain_syncq(sq, qp);
6644                 mutex_enter(SQLOCK(sq));
6645 
6646                 /* The queue is drained */
6647                 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6648                 qp->q_sqflags &= ~Q_SQDRAINING;
6649                 /*
6650                  * NOTE: After this point qp should not be used since it may be
6651                  * closed.
6652                  */
6653         }
6654 
6655         ASSERT(MUTEX_HELD(SQLOCK(sq)));
6656         flags = sq->sq_flags;
6657 
6658         /*
6659          * sq->sq_head cannot change because we hold the
6660          * sqlock. However, a thread CAN decide that it is no longer
6661          * going to drain that queue.  However, this should be due to
6662          * a GOAWAY state, and we should see that here.
6663          *
6664          * This loop is not very efficient. One solution may be adding a second
6665          * pointer to the "draining" queue, but it is difficult to do when
6666          * queues are inserted in the middle due to priority ordering. Another
6667          * possibility is to yank the queue out of the sq list and put it onto
6668          * the "draining list" and then put it back if it can't be drained.
6669          */
6670 
6671         ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6672             (type & SQ_CI) || sq->sq_head->q_draining);
6673 
6674         /* Drop SQ_EXCL for non-CIPUT perimeters */
6675         if (!(type & SQ_CIPUT))
6676                 flags &= ~SQ_EXCL;
6677         ASSERT((flags & SQ_EXCL) == 0);
6678 
6679         /* Wake up any waiters. */
6680         if (flags & SQ_WANTWAKEUP) {
6681                 flags &= ~SQ_WANTWAKEUP;
6682                 cv_broadcast(&sq->sq_wait);
6683         }
6684         if (flags & SQ_WANTEXWAKEUP) {
6685                 flags &= ~SQ_WANTEXWAKEUP;
6686                 cv_broadcast(&sq->sq_exitwait);
6687         }
6688         sq->sq_flags = flags;
6689 
6690         ASSERT(sq->sq_count != 0);
6691         /* Release our claim. */
6692         sq->sq_count--;
6693 
6694         if (bg_service) {
6695                 ASSERT(sq->sq_servcount != 0);
6696                 sq->sq_servcount--;
6697         }
6698 
6699         mutex_exit(SQLOCK(sq));
6700 
6701         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6702             "drain_syncq end:%p", sq);
6703 }
6704 
6705 
6706 /*
6707  *
6708  * qdrain_syncq can be called (currently) from only one of two places:
6709  *      drain_syncq
6710  *      putnext  (or some variation of it).
6711  * and eventually
6712  *      qwait(_sig)
6713  *
6714  * If called from drain_syncq, we found it in the list of queues needing
6715  * service, so there is work to be done (or it wouldn't be in the list).
6716  *
6717  * If called from some putnext variation, it was because the
6718  * perimeter is open, but messages are blocking a putnext and
6719  * there is not a thread working on it.  Now a thread could start
6720  * working on it while we are getting ready to do so ourself, but
6721  * the thread would set the q_draining flag, and we can spin out.
6722  *
6723  * As for qwait(_sig), I think I shall let it continue to call
6724  * drain_syncq directly (after all, it will get here eventually).
6725  *
6726  * qdrain_syncq has to terminate when:
6727  * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6728  * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6729  *
6730  * ASSUMES:
6731  *      One claim
6732  *      QLOCK held
6733  *      SQLOCK not held
6734  *      Will release QLOCK before returning
6735  */
6736 void
6737 qdrain_syncq(syncq_t *sq, queue_t *q)
6738 {
6739         mblk_t          *bp;
6740 #ifdef DEBUG
6741         uint16_t        count;
6742 #endif
6743 
6744         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6745             "drain_syncq start:%p", sq);
6746         ASSERT(q->q_syncq == sq);
6747         ASSERT(MUTEX_HELD(QLOCK(q)));
6748         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6749         /*
6750          * For non-CIPUT perimeters, we should be called with the exclusive bit
6751          * set already. For CIPUT perimeters, we will be doing a concurrent
6752          * drain, so it better not be set.
6753          */
6754         ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6755         ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6756         ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6757         /*
6758          * All outer pointers are set, or none of them are
6759          */
6760         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6761             sq->sq_oprev == NULL) ||
6762             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6763             sq->sq_oprev != NULL));
6764 #ifdef DEBUG
6765         count = sq->sq_count;
6766         /*
6767          * This is OK without the putlocks, because we have one
6768          * claim either from the sq_count, or a putcount.  We could
6769          * get an erroneous value from other counts, but ours won't
6770          * change, so one way or another, we will have at least a
6771          * value of one.
6772          */
6773         SUM_SQ_PUTCOUNTS(sq, count);
6774         ASSERT(count >= 1);
6775 #endif /* DEBUG */
6776 
6777         /*
6778          * The first thing to do is find out if a thread is already draining
6779          * this queue. If so, we are done, just return.
6780          */
6781         if (q->q_draining) {
6782                 mutex_exit(QLOCK(q));
6783                 return;
6784         }
6785 
6786         /*
6787          * If the perimeter is exclusive, there is nothing we can do right now,
6788          * go away. Note that there is nothing to prevent this case from
6789          * changing right after this check, but the spin-out will catch it.
6790          */
6791 
6792         /* Tell other threads that we are draining this queue */
6793         q->q_draining = 1;   /* Protected by QLOCK */
6794 
6795         /*
6796          * If there is nothing to do, clear QFULL as necessary. This caters for
6797          * the case where an empty queue was enqueued onto the syncq.
6798          */
6799         if (q->q_sqhead == NULL) {
6800                 ASSERT(q->q_syncqmsgs == 0);
6801                 mutex_exit(QLOCK(q));
6802                 clr_qfull(q);
6803                 mutex_enter(QLOCK(q));
6804         }
6805 
6806         /*
6807          * Note that q_sqhead must be re-checked here in case another message
6808          * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6809          */
6810         for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6811                 /*
6812                  * Because we can enter this routine just because a putnext is
6813                  * blocked, we need to spin out if the perimeter wants to go
6814                  * exclusive as well as just blocked. We need to spin out also
6815                  * if events are queued on the syncq.
6816                  * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6817                  * set it, and it can't become exclusive while we hold a claim.
6818                  */
6819                 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6820                         break;
6821                 }
6822 
6823 #ifdef DEBUG
6824                 /*
6825                  * Since we are in qdrain_syncq, we already know the queue,
6826                  * but for sanity, we want to check this against the qp that
6827                  * was passed in by bp->b_queue.
6828                  */
6829 
6830                 ASSERT(bp->b_queue == q);
6831                 ASSERT(bp->b_queue->q_syncq == sq);
6832                 bp->b_queue = NULL;
6833 
6834                 /*
6835                  * We would have the following check in the DEBUG code:
6836                  *
6837                  * if (bp->b_prev != NULL)  {
6838                  *      ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6839                  * }
6840                  *
6841                  * This can't be done, however, since IP modifies qinfo
6842                  * structure at run-time (switching between IPv4 qinfo and IPv6
6843                  * qinfo), invalidating the check.
6844                  * So the assignment to func is left here, but the ASSERT itself
6845                  * is removed until the whole issue is resolved.
6846                  */
6847 #endif
6848                 ASSERT(q->q_sqhead == bp);
6849                 q->q_sqhead = bp->b_next;
6850                 bp->b_prev = bp->b_next = NULL;
6851                 ASSERT(q->q_syncqmsgs > 0);
6852                 mutex_exit(QLOCK(q));
6853 
6854                 ASSERT(bp->b_datap->db_ref != 0);
6855 
6856                 (void) (*q->q_qinfo->qi_putp)(q, bp);
6857 
6858                 mutex_enter(QLOCK(q));
6859 
6860                 /*
6861                  * q_syncqmsgs should only be decremented after executing the
6862                  * put procedure to avoid message re-ordering. This is due to an
6863                  * optimisation in putnext() which can call the put procedure
6864                  * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6865                  * being set).
6866                  *
6867                  * We also need to clear QFULL in the next service procedure
6868                  * queue if this is the last message destined for that queue.
6869                  *
6870                  * It would make better sense to have some sort of tunable for
6871                  * the low water mark, but these semantics are not yet defined.
6872                  * So, alas, we use a constant.
6873                  */
6874                 if (--q->q_syncqmsgs == 0) {
6875                         mutex_exit(QLOCK(q));
6876                         clr_qfull(q);
6877                         mutex_enter(QLOCK(q));
6878                 }
6879 
6880                 /*
6881                  * Always clear SQ_EXCL when CIPUT in order to handle
6882                  * qwriter(INNER). The putp() can call qwriter and get exclusive
6883                  * access IFF this is the only claim. So, we need to test for
6884                  * this possibility, acquire the mutex and clear the bit.
6885                  */
6886                 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6887                         mutex_enter(SQLOCK(sq));
6888                         sq->sq_flags &= ~SQ_EXCL;
6889                         mutex_exit(SQLOCK(sq));
6890                 }
6891         }
6892 
6893         /*
6894          * We should either have no messages on this queue, or we were told to
6895          * goaway by a waiter (which we will wake up at the end of this
6896          * function).
6897          */
6898         ASSERT((q->q_sqhead == NULL) ||
6899             (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6900 
6901         ASSERT(MUTEX_HELD(QLOCK(q)));
6902         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6903 
6904         /* Remove the q from the syncq list if all the messages are drained. */
6905         if (q->q_sqhead == NULL) {
6906                 ASSERT(q->q_syncqmsgs == 0);
6907                 mutex_enter(SQLOCK(sq));
6908                 if (q->q_sqflags & Q_SQQUEUED)
6909                         SQRM_Q(sq, q);
6910                 mutex_exit(SQLOCK(sq));
6911                 /*
6912                  * Since the queue is removed from the list, reset its priority.
6913                  */
6914                 q->q_spri = 0;
6915         }
6916 
6917         /*
6918          * Remember, the q_draining flag is used to let another thread know
6919          * that there is a thread currently draining the messages for a queue.
6920          * Since we are now done with this queue (even if there may be messages
6921          * still there), we need to clear this flag so some thread will work on
6922          * it if needed.
6923          */
6924         ASSERT(q->q_draining);
6925         q->q_draining = 0;
6926 
6927         /* Called with a claim, so OK to drop all locks. */
6928         mutex_exit(QLOCK(q));
6929 
6930         TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6931             "drain_syncq end:%p", sq);
6932 }
6933 /* END OF QDRAIN_SYNCQ  */
6934 
6935 
6936 /*
6937  * This is the mate to qdrain_syncq, except that it is putting the message onto
6938  * the queue instead of draining. Since the message is destined for the queue
6939  * that is selected, there is no need to identify the function because the
6940  * message is intended for the put routine for the queue. For debug kernels,
6941  * this routine will do it anyway just in case.
6942  *
6943  * After the message is enqueued on the syncq, it calls putnext_tail()
6944  * which will schedule a background thread to actually process the message.
6945  *
6946  * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6947  * SQLOCK(sq) and QLOCK(q) are not held.
6948  */
6949 void
6950 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6951 {
6952         ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6953         ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6954         ASSERT(sq->sq_count > 0);
6955         ASSERT(q->q_syncq == sq);
6956         ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6957             sq->sq_oprev == NULL) ||
6958             (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6959             sq->sq_oprev != NULL));
6960 
6961         mutex_enter(QLOCK(q));
6962 
6963 #ifdef DEBUG
6964         /*
6965          * This is used for debug in the qfill_syncq/qdrain_syncq case
6966          * to trace the queue that the message is intended for.  Note
6967          * that the original use was to identify the queue and function
6968          * to call on the drain.  In the new syncq, we have the context
6969          * of the queue that we are draining, so call it's putproc and
6970          * don't rely on the saved values.  But for debug this is still
6971          * useful information.
6972          */
6973         mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6974         mp->b_queue = q;
6975         mp->b_next = NULL;
6976 #endif
6977         ASSERT(q->q_syncq == sq);
6978         /*
6979          * Enqueue the message on the list.
6980          * SQPUT_MP() accesses q_syncqmsgs.  We are already holding QLOCK to
6981          * protect it.  So it's ok to acquire SQLOCK after SQPUT_MP().
6982          */
6983         SQPUT_MP(q, mp);
6984         mutex_enter(SQLOCK(sq));
6985 
6986         /*
6987          * And queue on syncq for scheduling, if not already queued.
6988          * Note that we need the SQLOCK for this, and for testing flags
6989          * at the end to see if we will drain.  So grab it now, and
6990          * release it before we call qdrain_syncq or return.
6991          */
6992         if (!(q->q_sqflags & Q_SQQUEUED)) {
6993                 q->q_spri = curthread->t_pri;
6994                 SQPUT_Q(sq, q);
6995         }
6996 #ifdef DEBUG
6997         else {
6998                 /*
6999                  * All of these conditions MUST be true!
7000                  */
7001                 ASSERT(sq->sq_tail != NULL);
7002                 if (sq->sq_tail == sq->sq_head) {
7003                         ASSERT((q->q_sqprev == NULL) &&
7004                             (q->q_sqnext == NULL));
7005                 } else {
7006                         ASSERT((q->q_sqprev != NULL) ||
7007                             (q->q_sqnext != NULL));
7008                 }
7009                 ASSERT(sq->sq_flags & SQ_QUEUED);
7010                 ASSERT(q->q_syncqmsgs != 0);
7011                 ASSERT(q->q_sqflags & Q_SQQUEUED);
7012         }
7013 #endif
7014         mutex_exit(QLOCK(q));
7015         /*
7016          * SQLOCK is still held, so sq_count can be safely decremented.
7017          */
7018         sq->sq_count--;
7019 
7020         putnext_tail(sq, q, 0);
7021         /* Should not reference sq or q after this point. */
7022 }
7023 
7024 /*  End of qfill_syncq  */
7025 
7026 /*
7027  * Remove all messages from a syncq (if qp is NULL) or remove all messages
7028  * that would be put into qp by drain_syncq.
7029  * Used when deleting the syncq (qp == NULL) or when detaching
7030  * a queue (qp != NULL).
7031  * Return non-zero if one or more messages were freed.
7032  *
7033  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
7034  * sq_putlocks are used.
7035  *
7036  * NOTE: This function assumes that it is called from the close() context and
7037  * that all the queues in the syncq are going away. For this reason it doesn't
7038  * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
7039  * currently valid, but it is useful to rethink this function to behave properly
7040  * in other cases.
7041  */
7042 int
7043 flush_syncq(syncq_t *sq, queue_t *qp)
7044 {
7045         mblk_t          *bp, *mp_head, *mp_next, *mp_prev;
7046         queue_t         *q;
7047         int             ret = 0;
7048 
7049         mutex_enter(SQLOCK(sq));
7050 
7051         /*
7052          * Before we leave, we need to make sure there are no
7053          * events listed for this queue.  All events for this queue
7054          * will just be freed.
7055          */
7056         if (qp != NULL && sq->sq_evhead != NULL) {
7057                 ASSERT(sq->sq_flags & SQ_EVENTS);
7058 
7059                 mp_prev = NULL;
7060                 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
7061                         mp_next = bp->b_next;
7062                         if (bp->b_queue == qp) {
7063                                 /* Delete this message */
7064                                 if (mp_prev != NULL) {
7065                                         mp_prev->b_next = mp_next;
7066                                         /*
7067                                          * Update sq_evtail if the last element
7068                                          * is removed.
7069                                          */
7070                                         if (bp == sq->sq_evtail) {
7071                                                 ASSERT(mp_next == NULL);
7072                                                 sq->sq_evtail = mp_prev;
7073                                         }
7074                                 } else
7075                                         sq->sq_evhead = mp_next;
7076                                 if (sq->sq_evhead == NULL)
7077                                         sq->sq_flags &= ~SQ_EVENTS;
7078                                 bp->b_prev = bp->b_next = NULL;
7079                                 freemsg(bp);
7080                                 ret++;
7081                         } else {
7082                                 mp_prev = bp;
7083                         }
7084                 }
7085         }
7086 
7087         /*
7088          * Walk sq_head and:
7089          *      - match qp if qp is set, remove it's messages
7090          *      - all if qp is not set
7091          */
7092         q = sq->sq_head;
7093         while (q != NULL) {
7094                 ASSERT(q->q_syncq == sq);
7095                 if ((qp == NULL) || (qp == q)) {
7096                         /*
7097                          * Yank the messages as a list off the queue
7098                          */
7099                         mp_head = q->q_sqhead;
7100                         /*
7101                          * We do not have QLOCK(q) here (which is safe due to
7102                          * assumptions mentioned above). To obtain the lock we
7103                          * need to release SQLOCK which may allow lots of things
7104                          * to change upon us. This place requires more analysis.
7105                          */
7106                         q->q_sqhead = q->q_sqtail = NULL;
7107                         ASSERT(mp_head->b_queue &&
7108                             mp_head->b_queue->q_syncq == sq);
7109 
7110                         /*
7111                          * Free each of the messages.
7112                          */
7113                         for (bp = mp_head; bp != NULL; bp = mp_next) {
7114                                 mp_next = bp->b_next;
7115                                 bp->b_prev = bp->b_next = NULL;
7116                                 freemsg(bp);
7117                                 ret++;
7118                         }
7119                         /*
7120                          * Now remove the queue from the syncq.
7121                          */
7122                         ASSERT(q->q_sqflags & Q_SQQUEUED);
7123                         SQRM_Q(sq, q);
7124                         q->q_spri = 0;
7125                         q->q_syncqmsgs = 0;
7126 
7127                         /*
7128                          * If qp was specified, we are done with it and are
7129                          * going to drop SQLOCK(sq) and return. We wakeup syncq
7130                          * waiters while we still have the SQLOCK.
7131                          */
7132                         if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7133                                 sq->sq_flags &= ~SQ_WANTWAKEUP;
7134                                 cv_broadcast(&sq->sq_wait);
7135                         }
7136                         /* Drop SQLOCK across clr_qfull */
7137                         mutex_exit(SQLOCK(sq));
7138 
7139                         /*
7140                          * We avoid doing the test that drain_syncq does and
7141                          * unconditionally clear qfull for every flushed
7142                          * message. Since flush_syncq is only called during
7143                          * close this should not be a problem.
7144                          */
7145                         clr_qfull(q);
7146                         if (qp != NULL) {
7147                                 return (ret);
7148                         } else {
7149                                 mutex_enter(SQLOCK(sq));
7150                                 /*
7151                                  * The head was removed by SQRM_Q above.
7152                                  * reread the new head and flush it.
7153                                  */
7154                                 q = sq->sq_head;
7155                         }
7156                 } else {
7157                         q = q->q_sqnext;
7158                 }
7159                 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7160         }
7161 
7162         if (sq->sq_flags & SQ_WANTWAKEUP) {
7163                 sq->sq_flags &= ~SQ_WANTWAKEUP;
7164                 cv_broadcast(&sq->sq_wait);
7165         }
7166 
7167         mutex_exit(SQLOCK(sq));
7168         return (ret);
7169 }
7170 
7171 /*
7172  * Propagate all messages from a syncq to the next syncq that are associated
7173  * with the specified queue. If the queue is attached to a driver or if the
7174  * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7175  *
7176  * Assumes that the stream is strlock()'ed. We don't come here if there
7177  * are no messages to propagate.
7178  *
7179  * NOTE : If the queue is attached to a driver, all the messages are freed
7180  * as there is no point in propagating the messages from the driver syncq
7181  * to the closing stream head which will in turn get freed later.
7182  */
7183 static int
7184 propagate_syncq(queue_t *qp)
7185 {
7186         mblk_t          *bp, *head, *tail, *prev, *next;
7187         syncq_t         *sq;
7188         queue_t         *nqp;
7189         syncq_t         *nsq;
7190         boolean_t       isdriver;
7191         int             moved = 0;
7192         uint16_t        flags;
7193         pri_t           priority = curthread->t_pri;
7194 #ifdef DEBUG
7195         void            (*func)();
7196 #endif
7197 
7198         sq = qp->q_syncq;
7199         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7200         /* debug macro */
7201         SQ_PUTLOCKS_HELD(sq);
7202         /*
7203          * As entersq() does not increment the sq_count for
7204          * the write side, check sq_count for non-QPERQ
7205          * perimeters alone.
7206          */
7207         ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7208 
7209         /*
7210          * propagate_syncq() can be called because of either messages on the
7211          * queue syncq or because on events on the queue syncq. Do actual
7212          * message propagations if there are any messages.
7213          */
7214         if (qp->q_syncqmsgs) {
7215                 isdriver = (qp->q_flag & QISDRV);
7216 
7217                 if (!isdriver) {
7218                         nqp = qp->q_next;
7219                         nsq = nqp->q_syncq;
7220                         ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7221                         /* debug macro */
7222                         SQ_PUTLOCKS_HELD(nsq);
7223 #ifdef DEBUG
7224                         func = (void (*)())nqp->q_qinfo->qi_putp;
7225 #endif
7226                 }
7227 
7228                 SQRM_Q(sq, qp);
7229                 priority = MAX(qp->q_spri, priority);
7230                 qp->q_spri = 0;
7231                 head = qp->q_sqhead;
7232                 tail = qp->q_sqtail;
7233                 qp->q_sqhead = qp->q_sqtail = NULL;
7234                 qp->q_syncqmsgs = 0;
7235 
7236                 /*
7237                  * Walk the list of messages, and free them if this is a driver,
7238                  * otherwise reset the b_prev and b_queue value to the new putp.
7239                  * Afterward, we will just add the head to the end of the next
7240                  * syncq, and point the tail to the end of this one.
7241                  */
7242 
7243                 for (bp = head; bp != NULL; bp = next) {
7244                         next = bp->b_next;
7245                         if (isdriver) {
7246                                 bp->b_prev = bp->b_next = NULL;
7247                                 freemsg(bp);
7248                                 continue;
7249                         }
7250                         /* Change the q values for this message */
7251                         bp->b_queue = nqp;
7252 #ifdef DEBUG
7253                         bp->b_prev = (mblk_t *)func;
7254 #endif
7255                         moved++;
7256                 }
7257                 /*
7258                  * Attach list of messages to the end of the new queue (if there
7259                  * is a list of messages).
7260                  */
7261 
7262                 if (!isdriver && head != NULL) {
7263                         ASSERT(tail != NULL);
7264                         if (nqp->q_sqhead == NULL) {
7265                                 nqp->q_sqhead = head;
7266                         } else {
7267                                 ASSERT(nqp->q_sqtail != NULL);
7268                                 nqp->q_sqtail->b_next = head;
7269                         }
7270                         nqp->q_sqtail = tail;
7271                         /*
7272                          * When messages are moved from high priority queue to
7273                          * another queue, the destination queue priority is
7274                          * upgraded.
7275                          */
7276 
7277                         if (priority > nqp->q_spri)
7278                                 nqp->q_spri = priority;
7279 
7280                         SQPUT_Q(nsq, nqp);
7281 
7282                         nqp->q_syncqmsgs += moved;
7283                         ASSERT(nqp->q_syncqmsgs != 0);
7284                 }
7285         }
7286 
7287         /*
7288          * Before we leave, we need to make sure there are no
7289          * events listed for this queue.  All events for this queue
7290          * will just be freed.
7291          */
7292         if (sq->sq_evhead != NULL) {
7293                 ASSERT(sq->sq_flags & SQ_EVENTS);
7294                 prev = NULL;
7295                 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7296                         next = bp->b_next;
7297                         if (bp->b_queue == qp) {
7298                                 /* Delete this message */
7299                                 if (prev != NULL) {
7300                                         prev->b_next = next;
7301                                         /*
7302                                          * Update sq_evtail if the last element
7303                                          * is removed.
7304                                          */
7305                                         if (bp == sq->sq_evtail) {
7306                                                 ASSERT(next == NULL);
7307                                                 sq->sq_evtail = prev;
7308                                         }
7309                                 } else
7310                                         sq->sq_evhead = next;
7311                                 if (sq->sq_evhead == NULL)
7312                                         sq->sq_flags &= ~SQ_EVENTS;
7313                                 bp->b_prev = bp->b_next = NULL;
7314                                 freemsg(bp);
7315                         } else {
7316                                 prev = bp;
7317                         }
7318                 }
7319         }
7320 
7321         flags = sq->sq_flags;
7322 
7323         /* Wake up any waiter before leaving. */
7324         if (flags & SQ_WANTWAKEUP) {
7325                 flags &= ~SQ_WANTWAKEUP;
7326                 cv_broadcast(&sq->sq_wait);
7327         }
7328         sq->sq_flags = flags;
7329 
7330         return (moved);
7331 }
7332 
7333 /*
7334  * Try and upgrade to exclusive access at the inner perimeter. If this can
7335  * not be done without blocking then request will be queued on the syncq
7336  * and drain_syncq will run it later.
7337  *
7338  * This routine can only be called from put or service procedures plus
7339  * asynchronous callback routines that have properly entered the queue (with
7340  * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7341  * associated with q.
7342  */
7343 void
7344 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7345 {
7346         syncq_t *sq = q->q_syncq;
7347         uint16_t count;
7348 
7349         mutex_enter(SQLOCK(sq));
7350         count = sq->sq_count;
7351         SQ_PUTLOCKS_ENTER(sq);
7352         SUM_SQ_PUTCOUNTS(sq, count);
7353         ASSERT(count >= 1);
7354         ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7355 
7356         if (count == 1) {
7357                 /*
7358                  * Can upgrade. This case also handles nested qwriter calls
7359                  * (when the qwriter callback function calls qwriter). In that
7360                  * case SQ_EXCL is already set.
7361                  */
7362                 sq->sq_flags |= SQ_EXCL;
7363                 SQ_PUTLOCKS_EXIT(sq);
7364                 mutex_exit(SQLOCK(sq));
7365                 (*func)(q, mp);
7366                 /*
7367                  * Assumes that leavesq, putnext, and drain_syncq will reset
7368                  * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7369                  * until putnext, leavesq, or drain_syncq drops it.
7370                  * That way we handle nested qwriter(INNER) without dropping
7371                  * SQ_EXCL until the outermost qwriter callback routine is
7372                  * done.
7373                  */
7374                 return;
7375         }
7376         SQ_PUTLOCKS_EXIT(sq);
7377         sqfill_events(sq, q, mp, func);
7378 }
7379 
7380 /*
7381  * Synchronous callback support functions
7382  */
7383 
7384 /*
7385  * Allocate a callback parameter structure.
7386  * Assumes that caller initializes the flags and the id.
7387  * Acquires SQLOCK(sq) if non-NULL is returned.
7388  */
7389 callbparams_t *
7390 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7391 {
7392         callbparams_t *cbp;
7393         size_t size = sizeof (callbparams_t);
7394 
7395         cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7396 
7397         /*
7398          * Only try tryhard allocation if the caller is ready to panic.
7399          * Otherwise just fail.
7400          */
7401         if (cbp == NULL) {
7402                 if (kmflags & KM_PANIC)
7403                         cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7404                             &size, kmflags);
7405                 else
7406                         return (NULL);
7407         }
7408 
7409         ASSERT(size >= sizeof (callbparams_t));
7410         cbp->cbp_size = size;
7411         cbp->cbp_sq = sq;
7412         cbp->cbp_func = func;
7413         cbp->cbp_arg = arg;
7414         mutex_enter(SQLOCK(sq));
7415         cbp->cbp_next = sq->sq_callbpend;
7416         sq->sq_callbpend = cbp;
7417         return (cbp);
7418 }
7419 
7420 void
7421 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7422 {
7423         callbparams_t **pp, *p;
7424 
7425         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7426 
7427         for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7428                 if (p == cbp) {
7429                         *pp = p->cbp_next;
7430                         kmem_free(p, p->cbp_size);
7431                         return;
7432                 }
7433         }
7434         (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7435             "callbparams_free: not found\n"));
7436 }
7437 
7438 void
7439 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7440 {
7441         callbparams_t **pp, *p;
7442 
7443         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7444 
7445         for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7446                 if (p->cbp_id == id && p->cbp_flags == flag) {
7447                         *pp = p->cbp_next;
7448                         kmem_free(p, p->cbp_size);
7449                         return;
7450                 }
7451         }
7452         (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7453             "callbparams_free_id: not found\n"));
7454 }
7455 
7456 /*
7457  * Callback wrapper function used by once-only callbacks that can be
7458  * cancelled (qtimeout and qbufcall)
7459  * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7460  * cancelled by the qun* functions.
7461  */
7462 void
7463 qcallbwrapper(void *arg)
7464 {
7465         callbparams_t *cbp = arg;
7466         syncq_t *sq;
7467         uint16_t count = 0;
7468         uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7469         uint16_t type;
7470 
7471         sq = cbp->cbp_sq;
7472         mutex_enter(SQLOCK(sq));
7473         type = sq->sq_type;
7474         if (!(type & SQ_CICB)) {
7475                 count = sq->sq_count;
7476                 SQ_PUTLOCKS_ENTER(sq);
7477                 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7478                 SUM_SQ_PUTCOUNTS(sq, count);
7479                 sq->sq_needexcl++;
7480                 ASSERT(sq->sq_needexcl != 0);        /* wraparound */
7481                 waitflags |= SQ_MESSAGES;
7482         }
7483         /* Can not handle exclusive entry at outer perimeter */
7484         ASSERT(type & SQ_COCB);
7485 
7486         while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7487                 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7488                     (sq->sq_cancelid == cbp->cbp_id)) {
7489                         /* timeout has been cancelled */
7490                         sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7491                         callbparams_free(sq, cbp);
7492                         if (!(type & SQ_CICB)) {
7493                                 ASSERT(sq->sq_needexcl > 0);
7494                                 sq->sq_needexcl--;
7495                                 if (sq->sq_needexcl == 0) {
7496                                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7497                                 }
7498                                 SQ_PUTLOCKS_EXIT(sq);
7499                         }
7500                         mutex_exit(SQLOCK(sq));
7501                         return;
7502                 }
7503                 sq->sq_flags |= SQ_WANTWAKEUP;
7504                 if (!(type & SQ_CICB)) {
7505                         SQ_PUTLOCKS_EXIT(sq);
7506                 }
7507                 cv_wait(&sq->sq_wait, SQLOCK(sq));
7508                 if (!(type & SQ_CICB)) {
7509                         count = sq->sq_count;
7510                         SQ_PUTLOCKS_ENTER(sq);
7511                         SUM_SQ_PUTCOUNTS(sq, count);
7512                 }
7513         }
7514 
7515         sq->sq_count++;
7516         ASSERT(sq->sq_count != 0);   /* Wraparound */
7517         if (!(type & SQ_CICB)) {
7518                 ASSERT(count == 0);
7519                 sq->sq_flags |= SQ_EXCL;
7520                 ASSERT(sq->sq_needexcl > 0);
7521                 sq->sq_needexcl--;
7522                 if (sq->sq_needexcl == 0) {
7523                         SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7524                 }
7525                 SQ_PUTLOCKS_EXIT(sq);
7526         }
7527 
7528         mutex_exit(SQLOCK(sq));
7529 
7530         cbp->cbp_func(cbp->cbp_arg);
7531 
7532         /*
7533          * We drop the lock only for leavesq to re-acquire it.
7534          * Possible optimization is inline of leavesq.
7535          */
7536         mutex_enter(SQLOCK(sq));
7537         callbparams_free(sq, cbp);
7538         mutex_exit(SQLOCK(sq));
7539         leavesq(sq, SQ_CALLBACK);
7540 }
7541 
7542 /*
7543  * No need to grab sq_putlocks here. See comment in strsubr.h that
7544  * explains when sq_putlocks are used.
7545  *
7546  * sq_count (or one of the sq_putcounts) has already been
7547  * decremented by the caller, and if SQ_QUEUED, we need to call
7548  * drain_syncq (the global syncq drain).
7549  * If putnext_tail is called with the SQ_EXCL bit set, we are in
7550  * one of two states, non-CIPUT perimeter, and we need to clear
7551  * it, or we went exclusive in the put procedure.  In any case,
7552  * we want to clear the bit now, and it is probably easier to do
7553  * this at the beginning of this function (remember, we hold
7554  * the SQLOCK).  Lastly, if there are other messages queued
7555  * on the syncq (and not for our destination), enable the syncq
7556  * for background work.
7557  */
7558 
7559 /* ARGSUSED */
7560 void
7561 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7562 {
7563         uint16_t        flags = sq->sq_flags;
7564 
7565         ASSERT(MUTEX_HELD(SQLOCK(sq)));
7566         ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7567 
7568         /* Clear SQ_EXCL if set in passflags */
7569         if (passflags & SQ_EXCL) {
7570                 flags &= ~SQ_EXCL;
7571         }
7572         if (flags & SQ_WANTWAKEUP) {
7573                 flags &= ~SQ_WANTWAKEUP;
7574                 cv_broadcast(&sq->sq_wait);
7575         }
7576         if (flags & SQ_WANTEXWAKEUP) {
7577                 flags &= ~SQ_WANTEXWAKEUP;
7578                 cv_broadcast(&sq->sq_exitwait);
7579         }
7580         sq->sq_flags = flags;
7581 
7582         /*
7583          * We have cleared SQ_EXCL if we were asked to, and started
7584          * the wakeup process for waiters.  If there are no writers
7585          * then we need to drain the syncq if we were told to, or
7586          * enable the background thread to do it.
7587          */
7588         if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7589                 if ((passflags & SQ_QUEUED) ||
7590                     (sq->sq_svcflags & SQ_DISABLED)) {
7591                         /* drain_syncq will take care of events in the list */
7592                         drain_syncq(sq);
7593                         return;
7594                 } else if (flags & SQ_QUEUED) {
7595                         sqenable(sq);
7596                 }
7597         }
7598         /* Drop the SQLOCK on exit */
7599         mutex_exit(SQLOCK(sq));
7600         TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7601             "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7602 }
7603 
7604 void
7605 set_qend(queue_t *q)
7606 {
7607         mutex_enter(QLOCK(q));
7608         if (!O_SAMESTR(q))
7609                 q->q_flag |= QEND;
7610         else
7611                 q->q_flag &= ~QEND;
7612         mutex_exit(QLOCK(q));
7613         q = _OTHERQ(q);
7614         mutex_enter(QLOCK(q));
7615         if (!O_SAMESTR(q))
7616                 q->q_flag |= QEND;
7617         else
7618                 q->q_flag &= ~QEND;
7619         mutex_exit(QLOCK(q));
7620 }
7621 
7622 /*
7623  * Set QFULL in next service procedure queue (that cares) if not already
7624  * set and if there are already more messages on the syncq than
7625  * sq_max_size.  If sq_max_size is 0, no flow control will be asserted on
7626  * any syncq.
7627  *
7628  * The fq here is the next queue with a service procedure.  This is where
7629  * we would fail canputnext, so this is where we need to set QFULL.
7630  * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7631  *
7632  * We already have QLOCK at this point. To avoid cross-locks with
7633  * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7634  * SQLOCK and sd_reflock, we need to drop respective locks first.
7635  */
7636 void
7637 set_qfull(queue_t *q)
7638 {
7639         queue_t         *fq = NULL;
7640 
7641         ASSERT(MUTEX_HELD(QLOCK(q)));
7642         if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7643             (q->q_syncqmsgs > sq_max_size)) {
7644                 if ((fq = q->q_nfsrv) == q) {
7645                         fq->q_flag |= QFULL;
7646                 } else {
7647                         mutex_exit(QLOCK(q));
7648                         mutex_enter(QLOCK(fq));
7649                         fq->q_flag |= QFULL;
7650                         mutex_exit(QLOCK(fq));
7651                         mutex_enter(QLOCK(q));
7652                 }
7653         }
7654 }
7655 
7656 void
7657 clr_qfull(queue_t *q)
7658 {
7659         queue_t *oq = q;
7660 
7661         q = q->q_nfsrv;
7662         /* Fast check if there is any work to do before getting the lock. */
7663         if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7664                 return;
7665         }
7666 
7667         /*
7668          * Do not reset QFULL (and backenable) if the q_count is the reason
7669          * for QFULL being set.
7670          */
7671         mutex_enter(QLOCK(q));
7672         /*
7673          * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7674          * Hence clear the QFULL.
7675          * If both q_count and q_mblkcnt are less than the hiwat mark,
7676          * clear the QFULL.
7677          */
7678         if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7679             (q->q_mblkcnt < q->q_hiwat))) {
7680                 q->q_flag &= ~QFULL;
7681                 /*
7682                  * A little more confusing, how about this way:
7683                  * if someone wants to write,
7684                  * AND
7685                  *    both counts are less than the lowat mark
7686                  *    OR
7687                  *    the lowat mark is zero
7688                  * THEN
7689                  * backenable
7690                  */
7691                 if ((q->q_flag & QWANTW) &&
7692                     (((q->q_count < q->q_lowat) &&
7693                     (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7694                         q->q_flag &= ~QWANTW;
7695                         mutex_exit(QLOCK(q));
7696                         backenable(oq, 0);
7697                 } else
7698                         mutex_exit(QLOCK(q));
7699         } else
7700                 mutex_exit(QLOCK(q));
7701 }
7702 
7703 /*
7704  * Set the forward service procedure pointer.
7705  *
7706  * Called at insert-time to cache a queue's next forward service procedure in
7707  * q_nfsrv; used by canput() and canputnext().  If the queue to be inserted
7708  * has a service procedure then q_nfsrv points to itself.  If the queue to be
7709  * inserted does not have a service procedure, then q_nfsrv points to the next
7710  * queue forward that has a service procedure.  If the queue is at the logical
7711  * end of the stream (driver for write side, stream head for the read side)
7712  * and does not have a service procedure, then q_nfsrv also points to itself.
7713  */
7714 void
7715 set_nfsrv_ptr(
7716         queue_t  *rnew,         /* read queue pointer to new module */
7717         queue_t  *wnew,         /* write queue pointer to new module */
7718         queue_t  *prev_rq,      /* read queue pointer to the module above */
7719         queue_t  *prev_wq)      /* write queue pointer to the module above */
7720 {
7721         queue_t *qp;
7722 
7723         if (prev_wq->q_next == NULL) {
7724                 /*
7725                  * Insert the driver, initialize the driver and stream head.
7726                  * In this case, prev_rq/prev_wq should be the stream head.
7727                  * _I_INSERT does not allow inserting a driver.  Make sure
7728                  * that it is not an insertion.
7729                  */
7730                 ASSERT(!(rnew->q_flag & _QINSERTING));
7731                 wnew->q_nfsrv = wnew;
7732                 if (rnew->q_qinfo->qi_srvp)
7733                         rnew->q_nfsrv = rnew;
7734                 else
7735                         rnew->q_nfsrv = prev_rq;
7736                 prev_rq->q_nfsrv = prev_rq;
7737                 prev_wq->q_nfsrv = prev_wq;
7738         } else {
7739                 /*
7740                  * set up read side q_nfsrv pointer.  This MUST be done
7741                  * before setting the write side, because the setting of
7742                  * the write side for a fifo may depend on it.
7743                  *
7744                  * Suppose we have a fifo that only has pipemod pushed.
7745                  * pipemod has no read or write service procedures, so
7746                  * nfsrv for both pipemod queues points to prev_rq (the
7747                  * stream read head).  Now push bufmod (which has only a
7748                  * read service procedure).  Doing the write side first,
7749                  * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7750                  * is WRONG; the next queue forward from wnew with a
7751                  * service procedure will be rnew, not the stream read head.
7752                  * Since the downstream queue (which in the case of a fifo
7753                  * is the read queue rnew) can affect upstream queues, it
7754                  * needs to be done first.  Setting up the read side first
7755                  * sets nfsrv for both pipemod queues to rnew and then
7756                  * when the write side is set up, wnew-q_nfsrv will also
7757                  * point to rnew.
7758                  */
7759                 if (rnew->q_qinfo->qi_srvp) {
7760                         /*
7761                          * use _OTHERQ() because, if this is a pipe, next
7762                          * module may have been pushed from other end and
7763                          * q_next could be a read queue.
7764                          */
7765                         qp = _OTHERQ(prev_wq->q_next);
7766                         while (qp && qp->q_nfsrv != qp) {
7767                                 qp->q_nfsrv = rnew;
7768                                 qp = backq(qp);
7769                         }
7770                         rnew->q_nfsrv = rnew;
7771                 } else
7772                         rnew->q_nfsrv = prev_rq->q_nfsrv;
7773 
7774                 /* set up write side q_nfsrv pointer */
7775                 if (wnew->q_qinfo->qi_srvp) {
7776                         wnew->q_nfsrv = wnew;
7777 
7778                         /*
7779                          * For insertion, need to update nfsrv of the modules
7780                          * above which do not have a service routine.
7781                          */
7782                         if (rnew->q_flag & _QINSERTING) {
7783                                 for (qp = prev_wq;
7784                                     qp != NULL && qp->q_nfsrv != qp;
7785                                     qp = backq(qp)) {
7786                                         qp->q_nfsrv = wnew->q_nfsrv;
7787                                 }
7788                         }
7789                 } else {
7790                         if (prev_wq->q_next == prev_rq)
7791                                 /*
7792                                  * Since prev_wq/prev_rq are the middle of a
7793                                  * fifo, wnew/rnew will also be the middle of
7794                                  * a fifo and wnew's nfsrv is same as rnew's.
7795                                  */
7796                                 wnew->q_nfsrv = rnew->q_nfsrv;
7797                         else
7798                                 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7799                 }
7800         }
7801 }
7802 
7803 /*
7804  * Reset the forward service procedure pointer; called at remove-time.
7805  */
7806 void
7807 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7808 {
7809         queue_t *tmp_qp;
7810 
7811         /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7812         if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7813                 for (tmp_qp = backq(wqp);
7814                     tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7815                     tmp_qp = backq(tmp_qp)) {
7816                         tmp_qp->q_nfsrv = wqp->q_nfsrv;
7817                 }
7818         }
7819 
7820         /* reset the read side q_nfsrv pointer */
7821         if (rqp->q_qinfo->qi_srvp) {
7822                 if (wqp->q_next) {   /* non-driver case */
7823                         tmp_qp = _OTHERQ(wqp->q_next);
7824                         while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7825                                 /* Note that rqp->q_next cannot be NULL */
7826                                 ASSERT(rqp->q_next != NULL);
7827                                 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7828                                 tmp_qp = backq(tmp_qp);
7829                         }
7830                 }
7831         }
7832 }
7833 
7834 /*
7835  * This routine should be called after all stream geometry changes to update
7836  * the stream head cached struio() rd/wr queue pointers. Note must be called
7837  * with the streamlock()ed.
7838  *
7839  * Note: only enables Synchronous STREAMS for a side of a Stream which has
7840  *       an explicit synchronous barrier module queue. That is, a queue that
7841  *       has specified a struio() type.
7842  */
7843 static void
7844 strsetuio(stdata_t *stp)
7845 {
7846         queue_t *wrq;
7847 
7848         if (stp->sd_flag & STPLEX) {
7849                 /*
7850                  * Not streamhead, but a mux, so no Synchronous STREAMS.
7851                  */
7852                 stp->sd_struiowrq = NULL;
7853                 stp->sd_struiordq = NULL;
7854                 return;
7855         }
7856         /*
7857          * Scan the write queue(s) while synchronous
7858          * until we find a qinfo uio type specified.
7859          */
7860         wrq = stp->sd_wrq->q_next;
7861         while (wrq) {
7862                 if (wrq->q_struiot == STRUIOT_NONE) {
7863                         wrq = 0;
7864                         break;
7865                 }
7866                 if (wrq->q_struiot != STRUIOT_DONTCARE)
7867                         break;
7868                 if (! _SAMESTR(wrq)) {
7869                         wrq = 0;
7870                         break;
7871                 }
7872                 wrq = wrq->q_next;
7873         }
7874         stp->sd_struiowrq = wrq;
7875         /*
7876          * Scan the read queue(s) while synchronous
7877          * until we find a qinfo uio type specified.
7878          */
7879         wrq = stp->sd_wrq->q_next;
7880         while (wrq) {
7881                 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7882                         wrq = 0;
7883                         break;
7884                 }
7885                 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7886                         break;
7887                 if (! _SAMESTR(wrq)) {
7888                         wrq = 0;
7889                         break;
7890                 }
7891                 wrq = wrq->q_next;
7892         }
7893         stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7894 }
7895 
7896 /*
7897  * pass_wput, unblocks the passthru queues, so that
7898  * messages can arrive at muxs lower read queue, before
7899  * I_LINK/I_UNLINK is acked/nacked.
7900  */
7901 static void
7902 pass_wput(queue_t *q, mblk_t *mp)
7903 {
7904         syncq_t *sq;
7905 
7906         sq = _RD(q)->q_syncq;
7907         if (sq->sq_flags & SQ_BLOCKED)
7908                 unblocksq(sq, SQ_BLOCKED, 0);
7909         putnext(q, mp);
7910 }
7911 
7912 /*
7913  * Set up queues for the link/unlink.
7914  * Create a new queue and block it and then insert it
7915  * below the stream head on the lower stream.
7916  * This prevents any messages from arriving during the setq
7917  * as well as while the mux is processing the LINK/I_UNLINK.
7918  * The blocked passq is unblocked once the LINK/I_UNLINK has
7919  * been acked or nacked or if a message is generated and sent
7920  * down muxs write put procedure.
7921  * See pass_wput().
7922  *
7923  * After the new queue is inserted, all messages coming from below are
7924  * blocked. The call to strlock will ensure that all activity in the stream head
7925  * read queue syncq is stopped (sq_count drops to zero).
7926  */
7927 static queue_t *
7928 link_addpassthru(stdata_t *stpdown)
7929 {
7930         queue_t *passq;
7931         sqlist_t sqlist;
7932 
7933         passq = allocq();
7934         STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7935         /* setq might sleep in allocator - avoid holding locks. */
7936         setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7937             SQ_CI|SQ_CO, B_FALSE);
7938         claimq(passq);
7939         blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7940         insertq(STREAM(passq), passq);
7941 
7942         /*
7943          * Use strlock() to wait for the stream head sq_count to drop to zero
7944          * since we are going to change q_ptr in the stream head.  Note that
7945          * insertq() doesn't wait for any syncq counts to drop to zero.
7946          */
7947         sqlist.sqlist_head = NULL;
7948         sqlist.sqlist_index = 0;
7949         sqlist.sqlist_size = sizeof (sqlist_t);
7950         sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7951         strlock(stpdown, &sqlist);
7952         strunlock(stpdown, &sqlist);
7953 
7954         releaseq(passq);
7955         return (passq);
7956 }
7957 
7958 /*
7959  * Let messages flow up into the mux by removing
7960  * the passq.
7961  */
7962 static void
7963 link_rempassthru(queue_t *passq)
7964 {
7965         claimq(passq);
7966         removeq(passq);
7967         releaseq(passq);
7968         freeq(passq);
7969 }
7970 
7971 /*
7972  * Wait for the condition variable pointed to by `cvp' to be signaled,
7973  * or for `tim' milliseconds to elapse, whichever comes first.  If `tim'
7974  * is negative, then there is no time limit.  If `nosigs' is non-zero,
7975  * then the wait will be non-interruptible.
7976  *
7977  * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7978  */
7979 clock_t
7980 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7981 {
7982         clock_t ret;
7983 
7984         if (tim < 0) {
7985                 if (nosigs) {
7986                         cv_wait(cvp, mp);
7987                         ret = 1;
7988                 } else {
7989                         ret = cv_wait_sig(cvp, mp);
7990                 }
7991         } else if (tim > 0) {
7992                 /*
7993                  * convert milliseconds to clock ticks
7994                  */
7995                 if (nosigs) {
7996                         ret = cv_reltimedwait(cvp, mp,
7997                             MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7998                 } else {
7999                         ret = cv_reltimedwait_sig(cvp, mp,
8000                             MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
8001                 }
8002         } else {
8003                 ret = -1;
8004         }
8005         return (ret);
8006 }
8007 
8008 /*
8009  * Wait until the stream head can determine if it is at the mark but
8010  * don't wait forever to prevent a race condition between the "mark" state
8011  * in the stream head and any mark state in the caller/user of this routine.
8012  *
8013  * This is used by sockets and for a socket it would be incorrect
8014  * to return a failure for SIOCATMARK when there is no data in the receive
8015  * queue and the marked urgent data is traveling up the stream.
8016  *
8017  * This routine waits until the mark is known by waiting for one of these
8018  * three events:
8019  *      The stream head read queue becoming non-empty (including an EOF).
8020  *      The STRATMARK flag being set (due to a MSGMARKNEXT message).
8021  *      The STRNOTATMARK flag being set (which indicates that the transport
8022  *      has sent a MSGNOTMARKNEXT message to indicate that it is not at
8023  *      the mark).
8024  *
8025  * The routine returns 1 if the stream is at the mark; 0 if it can
8026  * be determined that the stream is not at the mark.
8027  * If the wait times out and it can't determine
8028  * whether or not the stream might be at the mark the routine will return -1.
8029  *
8030  * Note: This routine should only be used when a mark is pending i.e.,
8031  * in the socket case the SIGURG has been posted.
8032  * Note2: This can not wakeup just because synchronous streams indicate
8033  * that data is available since it is not possible to use the synchronous
8034  * streams interfaces to determine the b_flag value for the data queued below
8035  * the stream head.
8036  */
8037 int
8038 strwaitmark(vnode_t *vp)
8039 {
8040         struct stdata *stp = vp->v_stream;
8041         queue_t *rq = _RD(stp->sd_wrq);
8042         int mark;
8043 
8044         mutex_enter(&stp->sd_lock);
8045         while (rq->q_first == NULL &&
8046             !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
8047                 stp->sd_flag |= RSLEEP;
8048 
8049                 /* Wait for 100 milliseconds for any state change. */
8050                 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
8051                         mutex_exit(&stp->sd_lock);
8052                         return (-1);
8053                 }
8054         }
8055         if (stp->sd_flag & STRATMARK)
8056                 mark = 1;
8057         else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
8058                 mark = 1;
8059         else
8060                 mark = 0;
8061 
8062         mutex_exit(&stp->sd_lock);
8063         return (mark);
8064 }
8065 
8066 /*
8067  * Set a read side error. If persist is set change the socket error
8068  * to persistent. If errfunc is set install the function as the exported
8069  * error handler.
8070  */
8071 void
8072 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8073 {
8074         struct stdata *stp = vp->v_stream;
8075 
8076         mutex_enter(&stp->sd_lock);
8077         stp->sd_rerror = error;
8078         if (error == 0 && errfunc == NULL)
8079                 stp->sd_flag &= ~STRDERR;
8080         else
8081                 stp->sd_flag |= STRDERR;
8082         if (persist) {
8083                 stp->sd_flag &= ~STRDERRNONPERSIST;
8084         } else {
8085                 stp->sd_flag |= STRDERRNONPERSIST;
8086         }
8087         stp->sd_rderrfunc = errfunc;
8088         if (error != 0 || errfunc != NULL) {
8089                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);      /* readers */
8090                 cv_broadcast(&stp->sd_wrq->q_wait);           /* writers */
8091                 cv_broadcast(&stp->sd_monitor);                  /* ioctllers */
8092 
8093                 mutex_exit(&stp->sd_lock);
8094                 pollwakeup(&stp->sd_pollist, POLLERR);
8095                 mutex_enter(&stp->sd_lock);
8096 
8097                 if (stp->sd_sigflags & S_ERROR)
8098                         strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8099         }
8100         mutex_exit(&stp->sd_lock);
8101 }
8102 
8103 /*
8104  * Set a write side error. If persist is set change the socket error
8105  * to persistent.
8106  */
8107 void
8108 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8109 {
8110         struct stdata *stp = vp->v_stream;
8111 
8112         mutex_enter(&stp->sd_lock);
8113         stp->sd_werror = error;
8114         if (error == 0 && errfunc == NULL)
8115                 stp->sd_flag &= ~STWRERR;
8116         else
8117                 stp->sd_flag |= STWRERR;
8118         if (persist) {
8119                 stp->sd_flag &= ~STWRERRNONPERSIST;
8120         } else {
8121                 stp->sd_flag |= STWRERRNONPERSIST;
8122         }
8123         stp->sd_wrerrfunc = errfunc;
8124         if (error != 0 || errfunc != NULL) {
8125                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);      /* readers */
8126                 cv_broadcast(&stp->sd_wrq->q_wait);           /* writers */
8127                 cv_broadcast(&stp->sd_monitor);                  /* ioctllers */
8128 
8129                 mutex_exit(&stp->sd_lock);
8130                 pollwakeup(&stp->sd_pollist, POLLERR);
8131                 mutex_enter(&stp->sd_lock);
8132 
8133                 if (stp->sd_sigflags & S_ERROR)
8134                         strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8135         }
8136         mutex_exit(&stp->sd_lock);
8137 }
8138 
8139 /*
8140  * Make the stream return 0 (EOF) when all data has been read.
8141  * No effect on write side.
8142  */
8143 void
8144 strseteof(vnode_t *vp, int eof)
8145 {
8146         struct stdata *stp = vp->v_stream;
8147 
8148         mutex_enter(&stp->sd_lock);
8149         if (!eof) {
8150                 stp->sd_flag &= ~STREOF;
8151                 mutex_exit(&stp->sd_lock);
8152                 return;
8153         }
8154         stp->sd_flag |= STREOF;
8155         if (stp->sd_flag & RSLEEP) {
8156                 stp->sd_flag &= ~RSLEEP;
8157                 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8158         }
8159 
8160         mutex_exit(&stp->sd_lock);
8161         pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8162         mutex_enter(&stp->sd_lock);
8163 
8164         if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8165                 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8166         mutex_exit(&stp->sd_lock);
8167 }
8168 
8169 void
8170 strflushrq(vnode_t *vp, int flag)
8171 {
8172         struct stdata *stp = vp->v_stream;
8173 
8174         mutex_enter(&stp->sd_lock);
8175         flushq(_RD(stp->sd_wrq), flag);
8176         mutex_exit(&stp->sd_lock);
8177 }
8178 
8179 void
8180 strsetrputhooks(vnode_t *vp, uint_t flags, msgfunc_t protofunc,
8181     msgfunc_t miscfunc)
8182 {
8183         struct stdata *stp = vp->v_stream;
8184 
8185         mutex_enter(&stp->sd_lock);
8186 
8187         if (protofunc == NULL)
8188                 stp->sd_rprotofunc = strrput_proto;
8189         else
8190                 stp->sd_rprotofunc = protofunc;
8191 
8192         if (miscfunc == NULL)
8193                 stp->sd_rmiscfunc = strrput_misc;
8194         else
8195                 stp->sd_rmiscfunc = miscfunc;
8196 
8197         if (flags & SH_CONSOL_DATA)
8198                 stp->sd_rput_opt |= SR_CONSOL_DATA;
8199         else
8200                 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8201 
8202         if (flags & SH_SIGALLDATA)
8203                 stp->sd_rput_opt |= SR_SIGALLDATA;
8204         else
8205                 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8206 
8207         if (flags & SH_IGN_ZEROLEN)
8208                 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8209         else
8210                 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8211 
8212         mutex_exit(&stp->sd_lock);
8213 }
8214 
8215 void
8216 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8217 {
8218         struct stdata *stp = vp->v_stream;
8219 
8220         mutex_enter(&stp->sd_lock);
8221         stp->sd_closetime = closetime;
8222 
8223         if (flags & SH_SIGPIPE)
8224                 stp->sd_wput_opt |= SW_SIGPIPE;
8225         else
8226                 stp->sd_wput_opt &= ~SW_SIGPIPE;
8227         if (flags & SH_RECHECK_ERR)
8228                 stp->sd_wput_opt |= SW_RECHECK_ERR;
8229         else
8230                 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8231 
8232         mutex_exit(&stp->sd_lock);
8233 }
8234 
8235 void
8236 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8237 {
8238         struct stdata *stp = vp->v_stream;
8239 
8240         mutex_enter(&stp->sd_lock);
8241 
8242         stp->sd_rputdatafunc = rdatafunc;
8243         stp->sd_wputdatafunc = wdatafunc;
8244 
8245         mutex_exit(&stp->sd_lock);
8246 }
8247 
8248 /* Used within framework when the queue is already locked */
8249 void
8250 qenable_locked(queue_t *q)
8251 {
8252         stdata_t *stp = STREAM(q);
8253 
8254         ASSERT(MUTEX_HELD(QLOCK(q)));
8255 
8256         if (!q->q_qinfo->qi_srvp)
8257                 return;
8258 
8259         /*
8260          * Do not place on run queue if already enabled or closing.
8261          */
8262         if (q->q_flag & (QWCLOSE|QENAB))
8263                 return;
8264 
8265         /*
8266          * mark queue enabled and place on run list if it is not already being
8267          * serviced. If it is serviced, the runservice() function will detect
8268          * that QENAB is set and call service procedure before clearing
8269          * QINSERVICE flag.
8270          */
8271         q->q_flag |= QENAB;
8272         if (q->q_flag & QINSERVICE)
8273                 return;
8274 
8275         /* Record the time of qenable */
8276         q->q_qtstamp = ddi_get_lbolt();
8277 
8278         /*
8279          * Put the queue in the stp list and schedule it for background
8280          * processing if it is not already scheduled or if stream head does not
8281          * intent to process it in the foreground later by setting
8282          * STRS_WILLSERVICE flag.
8283          */
8284         mutex_enter(&stp->sd_qlock);
8285         /*
8286          * If there are already something on the list, stp flags should show
8287          * intention to drain it.
8288          */
8289         IMPLY(STREAM_NEEDSERVICE(stp),
8290             (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8291 
8292         ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8293         stp->sd_nqueues++;
8294 
8295         /*
8296          * If no one will drain this stream we are the first producer and
8297          * need to schedule it for background thread.
8298          */
8299         if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8300                 /*
8301                  * No one will service this stream later, so we have to
8302                  * schedule it now.
8303                  */
8304                 STRSTAT(stenables);
8305                 stp->sd_svcflags |= STRS_SCHEDULED;
8306                 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8307                     (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8308 
8309                 if (stp->sd_servid == NULL) {
8310                         /*
8311                          * Task queue failed so fail over to the backup
8312                          * servicing thread.
8313                          */
8314                         STRSTAT(taskqfails);
8315                         /*
8316                          * It is safe to clear STRS_SCHEDULED flag because it
8317                          * was set by this thread above.
8318                          */
8319                         stp->sd_svcflags &= ~STRS_SCHEDULED;
8320 
8321                         /*
8322                          * Failover scheduling is protected by service_queue
8323                          * lock.
8324                          */
8325                         mutex_enter(&service_queue);
8326                         ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8327                         ASSERT(q->q_link == NULL);
8328                         /*
8329                          * Append the queue to qhead/qtail list.
8330                          */
8331                         if (qhead == NULL)
8332                                 qhead = q;
8333                         else
8334                                 qtail->q_link = q;
8335                         qtail = q;
8336                         /*
8337                          * Clear stp queue list.
8338                          */
8339                         stp->sd_qhead = stp->sd_qtail = NULL;
8340                         stp->sd_nqueues = 0;
8341                         /*
8342                          * Wakeup background queue processing thread.
8343                          */
8344                         cv_signal(&services_to_run);
8345                         mutex_exit(&service_queue);
8346                 }
8347         }
8348         mutex_exit(&stp->sd_qlock);
8349 }
8350 
8351 static void
8352 queue_service(queue_t *q)
8353 {
8354         /*
8355          * The queue in the list should have
8356          * QENAB flag set and should not have
8357          * QINSERVICE flag set. QINSERVICE is
8358          * set when the queue is dequeued and
8359          * qenable_locked doesn't enqueue a
8360          * queue with QINSERVICE set.
8361          */
8362 
8363         ASSERT(!(q->q_flag & QINSERVICE));
8364         ASSERT((q->q_flag & QENAB));
8365         mutex_enter(QLOCK(q));
8366         q->q_flag &= ~QENAB;
8367         q->q_flag |= QINSERVICE;
8368         mutex_exit(QLOCK(q));
8369         runservice(q);
8370 }
8371 
8372 static void
8373 syncq_service(syncq_t *sq)
8374 {
8375         STRSTAT(syncqservice);
8376         mutex_enter(SQLOCK(sq));
8377         ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8378         ASSERT(sq->sq_servcount != 0);
8379         ASSERT(sq->sq_next == NULL);
8380 
8381         /* if we came here from the background thread, clear the flag */
8382         if (sq->sq_svcflags & SQ_BGTHREAD)
8383                 sq->sq_svcflags &= ~SQ_BGTHREAD;
8384 
8385         /* let drain_syncq know that it's being called in the background */
8386         sq->sq_svcflags |= SQ_SERVICE;
8387         drain_syncq(sq);
8388 }
8389 
8390 static void
8391 qwriter_outer_service(syncq_t *outer)
8392 {
8393         /*
8394          * Note that SQ_WRITER is used on the outer perimeter
8395          * to signal that a qwriter(OUTER) is either investigating
8396          * running or that it is actually running a function.
8397          */
8398         outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8399 
8400         /*
8401          * All inner syncq are empty and have SQ_WRITER set
8402          * to block entering the outer perimeter.
8403          *
8404          * We do not need to explicitly call write_now since
8405          * outer_exit does it for us.
8406          */
8407         outer_exit(outer);
8408 }
8409 
8410 static void
8411 mblk_free(mblk_t *mp)
8412 {
8413         dblk_t *dbp = mp->b_datap;
8414         frtn_t *frp = dbp->db_frtnp;
8415 
8416         mp->b_next = NULL;
8417         if (dbp->db_fthdr != NULL)
8418                 str_ftfree(dbp);
8419 
8420         ASSERT(dbp->db_fthdr == NULL);
8421         frp->free_func(frp->free_arg);
8422         ASSERT(dbp->db_mblk == mp);
8423 
8424         if (dbp->db_credp != NULL) {
8425                 crfree(dbp->db_credp);
8426                 dbp->db_credp = NULL;
8427         }
8428         dbp->db_cpid = -1;
8429         dbp->db_struioflag = 0;
8430         dbp->db_struioun.cksum.flags = 0;
8431 
8432         kmem_cache_free(dbp->db_cache, dbp);
8433 }
8434 
8435 /*
8436  * Background processing of the stream queue list.
8437  */
8438 static void
8439 stream_service(stdata_t *stp)
8440 {
8441         queue_t *q;
8442 
8443         mutex_enter(&stp->sd_qlock);
8444 
8445         STR_SERVICE(stp, q);
8446 
8447         stp->sd_svcflags &= ~STRS_SCHEDULED;
8448         stp->sd_servid = NULL;
8449         cv_signal(&stp->sd_qcv);
8450         mutex_exit(&stp->sd_qlock);
8451 }
8452 
8453 /*
8454  * Foreground processing of the stream queue list.
8455  */
8456 void
8457 stream_runservice(stdata_t *stp)
8458 {
8459         queue_t *q;
8460 
8461         mutex_enter(&stp->sd_qlock);
8462         STRSTAT(rservice);
8463         /*
8464          * We are going to drain this stream queue list, so qenable_locked will
8465          * not schedule it until we finish.
8466          */
8467         stp->sd_svcflags |= STRS_WILLSERVICE;
8468 
8469         STR_SERVICE(stp, q);
8470 
8471         stp->sd_svcflags &= ~STRS_WILLSERVICE;
8472         mutex_exit(&stp->sd_qlock);
8473         /*
8474          * Help backup background thread to drain the qhead/qtail list.
8475          */
8476         while (qhead != NULL) {
8477                 STRSTAT(qhelps);
8478                 mutex_enter(&service_queue);
8479                 DQ(q, qhead, qtail, q_link);
8480                 mutex_exit(&service_queue);
8481                 if (q != NULL)
8482                         queue_service(q);
8483         }
8484 }
8485 
8486 void
8487 stream_willservice(stdata_t *stp)
8488 {
8489         mutex_enter(&stp->sd_qlock);
8490         stp->sd_svcflags |= STRS_WILLSERVICE;
8491         mutex_exit(&stp->sd_qlock);
8492 }
8493 
8494 /*
8495  * Replace the cred currently in the mblk with a different one.
8496  * Also update db_cpid.
8497  */
8498 void
8499 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8500 {
8501         dblk_t *dbp = mp->b_datap;
8502         cred_t *ocr = dbp->db_credp;
8503 
8504         ASSERT(cr != NULL);
8505 
8506         if (cr != ocr) {
8507                 crhold(dbp->db_credp = cr);
8508                 if (ocr != NULL)
8509                         crfree(ocr);
8510         }
8511         /* Don't overwrite with NOPID */
8512         if (cpid != NOPID)
8513                 dbp->db_cpid = cpid;
8514 }
8515 
8516 /*
8517  * If the src message has a cred, then replace the cred currently in the mblk
8518  * with it.
8519  * Also update db_cpid.
8520  */
8521 void
8522 mblk_copycred(mblk_t *mp, const mblk_t *src)
8523 {
8524         dblk_t *dbp = mp->b_datap;
8525         cred_t *cr, *ocr;
8526         pid_t cpid;
8527 
8528         cr = msg_getcred(src, &cpid);
8529         if (cr == NULL)
8530                 return;
8531 
8532         ocr = dbp->db_credp;
8533         if (cr != ocr) {
8534                 crhold(dbp->db_credp = cr);
8535                 if (ocr != NULL)
8536                         crfree(ocr);
8537         }
8538         /* Don't overwrite with NOPID */
8539         if (cpid != NOPID)
8540                 dbp->db_cpid = cpid;
8541 }
8542 
8543 int
8544 hcksum_assoc(mblk_t *mp,  multidata_t *mmd, pdesc_t *pd,
8545     uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8546     uint32_t flags, int km_flags)
8547 {
8548         int rc = 0;
8549 
8550         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8551         if (mp->b_datap->db_type == M_DATA) {
8552                 /* Associate values for M_DATA type */
8553                 DB_CKSUMSTART(mp) = (intptr_t)start;
8554                 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8555                 DB_CKSUMEND(mp) = (intptr_t)end;
8556                 DB_CKSUMFLAGS(mp) = flags;
8557                 DB_CKSUM16(mp) = (uint16_t)value;
8558 
8559         } else {
8560                 pattrinfo_t pa_info;
8561 
8562                 ASSERT(mmd != NULL);
8563 
8564                 pa_info.type = PATTR_HCKSUM;
8565                 pa_info.len = sizeof (pattr_hcksum_t);
8566 
8567                 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8568                         pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8569 
8570                         hck->hcksum_start_offset = start;
8571                         hck->hcksum_stuff_offset = stuff;
8572                         hck->hcksum_end_offset = end;
8573                         hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8574                         hck->hcksum_flags = flags;
8575                 } else {
8576                         rc = -1;
8577                 }
8578         }
8579         return (rc);
8580 }
8581 
8582 void
8583 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8584     uint32_t *start, uint32_t *stuff, uint32_t *end,
8585     uint32_t *value, uint32_t *flags)
8586 {
8587         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8588         if (mp->b_datap->db_type == M_DATA) {
8589                 if (flags != NULL) {
8590                         *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8591                         if ((*flags & (HCK_PARTIALCKSUM |
8592                             HCK_FULLCKSUM)) != 0) {
8593                                 if (value != NULL)
8594                                         *value = (uint32_t)DB_CKSUM16(mp);
8595                                 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8596                                         if (start != NULL)
8597                                                 *start =
8598                                                     (uint32_t)DB_CKSUMSTART(mp);
8599                                         if (stuff != NULL)
8600                                                 *stuff =
8601                                                     (uint32_t)DB_CKSUMSTUFF(mp);
8602                                         if (end != NULL)
8603                                                 *end =
8604                                                     (uint32_t)DB_CKSUMEND(mp);
8605                                 }
8606                         }
8607                 }
8608         } else {
8609                 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8610 
8611                 ASSERT(mmd != NULL);
8612 
8613                 /* get hardware checksum attribute */
8614                 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8615                         pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8616 
8617                         ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8618                         if (flags != NULL)
8619                                 *flags = hck->hcksum_flags;
8620                         if (start != NULL)
8621                                 *start = hck->hcksum_start_offset;
8622                         if (stuff != NULL)
8623                                 *stuff = hck->hcksum_stuff_offset;
8624                         if (end != NULL)
8625                                 *end = hck->hcksum_end_offset;
8626                         if (value != NULL)
8627                                 *value = (uint32_t)
8628                                     hck->hcksum_cksum_val.inet_cksum;
8629                 }
8630         }
8631 }
8632 
8633 void
8634 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8635 {
8636         ASSERT(DB_TYPE(mp) == M_DATA);
8637         ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8638 
8639         /* Set the flags */
8640         DB_LSOFLAGS(mp) |= flags;
8641         DB_LSOMSS(mp) = mss;
8642 }
8643 
8644 void
8645 lso_info_cleanup(mblk_t *mp)
8646 {
8647         ASSERT(DB_TYPE(mp) == M_DATA);
8648 
8649         /* Clear the flags */
8650         DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8651         DB_LSOMSS(mp) = 0;
8652 }
8653 
8654 /*
8655  * Checksum buffer *bp for len bytes with psum partial checksum,
8656  * or 0 if none, and return the 16 bit partial checksum.
8657  */
8658 unsigned
8659 bcksum(uchar_t *bp, int len, unsigned int psum)
8660 {
8661         int odd = len & 1;
8662         extern unsigned int ip_ocsum();
8663 
8664         if (((intptr_t)bp & 1) == 0 && !odd) {
8665                 /*
8666                  * Bp is 16 bit aligned and len is multiple of 16 bit word.
8667                  */
8668                 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8669         }
8670         if (((intptr_t)bp & 1) != 0) {
8671                 /*
8672                  * Bp isn't 16 bit aligned.
8673                  */
8674                 unsigned int tsum;
8675 
8676 #ifdef _LITTLE_ENDIAN
8677                 psum += *bp;
8678 #else
8679                 psum += *bp << 8;
8680 #endif
8681                 len--;
8682                 bp++;
8683                 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8684                 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8685                 if (len & 1) {
8686                         bp += len - 1;
8687 #ifdef _LITTLE_ENDIAN
8688                         psum += *bp << 8;
8689 #else
8690                         psum += *bp;
8691 #endif
8692                 }
8693         } else {
8694                 /*
8695                  * Bp is 16 bit aligned.
8696                  */
8697                 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8698                 if (odd) {
8699                         bp += len - 1;
8700 #ifdef _LITTLE_ENDIAN
8701                         psum += *bp;
8702 #else
8703                         psum += *bp << 8;
8704 #endif
8705                 }
8706         }
8707         /*
8708          * Normalize psum to 16 bits before returning the new partial
8709          * checksum. The max psum value before normalization is 0x3FDFE.
8710          */
8711         return ((psum >> 16) + (psum & 0xFFFF));
8712 }
8713 
8714 boolean_t
8715 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8716 {
8717         boolean_t rc;
8718 
8719         ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8720         if (DB_TYPE(mp) == M_DATA) {
8721                 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8722         } else {
8723                 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8724 
8725                 ASSERT(mmd != NULL);
8726                 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8727         }
8728         return (rc);
8729 }
8730 
8731 void
8732 freemsgchain(mblk_t *mp)
8733 {
8734         mblk_t  *next;
8735 
8736         while (mp != NULL) {
8737                 next = mp->b_next;
8738                 mp->b_next = NULL;
8739 
8740                 freemsg(mp);
8741                 mp = next;
8742         }
8743 }
8744 
8745 mblk_t *
8746 copymsgchain(mblk_t *mp)
8747 {
8748         mblk_t  *nmp = NULL;
8749         mblk_t  **nmpp = &nmp;
8750 
8751         for (; mp != NULL; mp = mp->b_next) {
8752                 if ((*nmpp = copymsg(mp)) == NULL) {
8753                         freemsgchain(nmp);
8754                         return (NULL);
8755                 }
8756 
8757                 nmpp = &((*nmpp)->b_next);
8758         }
8759 
8760         return (nmp);
8761 }
8762 
8763 /* NOTE: Do not add code after this point. */
8764 #undef QLOCK
8765 
8766 /*
8767  * Replacement for QLOCK macro for those that can't use it.
8768  */
8769 kmutex_t *
8770 QLOCK(queue_t *q)
8771 {
8772         return (&(q)->q_lock);
8773 }
8774 
8775 /*
8776  * Dummy runqueues/queuerun functions functions for backwards compatibility.
8777  */
8778 #undef runqueues
8779 void
8780 runqueues(void)
8781 {
8782 }
8783 
8784 #undef queuerun
8785 void
8786 queuerun(void)
8787 {
8788 }
8789 
8790 /*
8791  * Initialize the STR stack instance, which tracks autopush and persistent
8792  * links.
8793  */
8794 /* ARGSUSED */
8795 static void *
8796 str_stack_init(netstackid_t stackid, netstack_t *ns)
8797 {
8798         str_stack_t     *ss;
8799         int i;
8800 
8801         ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8802         ss->ss_netstack = ns;
8803 
8804         /*
8805          * set up autopush
8806          */
8807         sad_initspace(ss);
8808 
8809         /*
8810          * set up mux_node structures.
8811          */
8812         ss->ss_devcnt = devcnt;      /* In case it should change before free */
8813         ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8814             ss->ss_devcnt), KM_SLEEP);
8815         for (i = 0; i < ss->ss_devcnt; i++)
8816                 ss->ss_mux_nodes[i].mn_imaj = i;
8817         return (ss);
8818 }
8819 
8820 /*
8821  * Note: run at zone shutdown and not destroy so that the PLINKs are
8822  * gone by the time other cleanup happens from the destroy callbacks.
8823  */
8824 static void
8825 str_stack_shutdown(netstackid_t stackid, void *arg)
8826 {
8827         str_stack_t *ss = (str_stack_t *)arg;
8828         int i;
8829         cred_t *cr;
8830 
8831         cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8832         ASSERT(cr != NULL);
8833 
8834         /* Undo all the I_PLINKs for this zone */
8835         for (i = 0; i < ss->ss_devcnt; i++) {
8836                 struct mux_edge         *ep;
8837                 ldi_handle_t            lh;
8838                 ldi_ident_t             li;
8839                 int                     ret;
8840                 int                     rval;
8841                 dev_t                   rdev;
8842 
8843                 ep = ss->ss_mux_nodes[i].mn_outp;
8844                 if (ep == NULL)
8845                         continue;
8846                 ret = ldi_ident_from_major((major_t)i, &li);
8847                 if (ret != 0) {
8848                         continue;
8849                 }
8850                 rdev = ep->me_dev;
8851                 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8852                     cr, &lh, li);
8853                 if (ret != 0) {
8854                         ldi_ident_release(li);
8855                         continue;
8856                 }
8857 
8858                 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8859                     cr, &rval);
8860                 if (ret) {
8861                         (void) ldi_close(lh, FREAD|FWRITE, cr);
8862                         ldi_ident_release(li);
8863                         continue;
8864                 }
8865                 (void) ldi_close(lh, FREAD|FWRITE, cr);
8866 
8867                 /* Close layered handles */
8868                 ldi_ident_release(li);
8869         }
8870         crfree(cr);
8871 
8872         sad_freespace(ss);
8873 
8874         kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8875         ss->ss_mux_nodes = NULL;
8876 }
8877 
8878 /*
8879  * Free the structure; str_stack_shutdown did the other cleanup work.
8880  */
8881 /* ARGSUSED */
8882 static void
8883 str_stack_fini(netstackid_t stackid, void *arg)
8884 {
8885         str_stack_t     *ss = (str_stack_t *)arg;
8886 
8887         kmem_free(ss, sizeof (*ss));
8888 }