Print this page
XXXX adding PID information to netstat output
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/os/strsubr.c
+++ new/usr/src/uts/common/os/strsubr.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
22 22 /* All Rights Reserved */
23 23
24 24
25 25 /*
26 26 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
27 27 * Use is subject to license terms.
28 28 */
29 29
30 30 #include <sys/types.h>
31 31 #include <sys/sysmacros.h>
32 32 #include <sys/param.h>
33 33 #include <sys/errno.h>
34 34 #include <sys/signal.h>
35 35 #include <sys/proc.h>
36 36 #include <sys/conf.h>
37 37 #include <sys/cred.h>
38 38 #include <sys/user.h>
39 39 #include <sys/vnode.h>
40 40 #include <sys/file.h>
41 41 #include <sys/session.h>
42 42 #include <sys/stream.h>
43 43 #include <sys/strsubr.h>
44 44 #include <sys/stropts.h>
45 45 #include <sys/poll.h>
46 46 #include <sys/systm.h>
47 47 #include <sys/cpuvar.h>
48 48 #include <sys/uio.h>
49 49 #include <sys/cmn_err.h>
50 50 #include <sys/priocntl.h>
51 51 #include <sys/procset.h>
52 52 #include <sys/vmem.h>
53 53 #include <sys/bitmap.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/siginfo.h>
56 56 #include <sys/vtrace.h>
57 57 #include <sys/callb.h>
58 58 #include <sys/debug.h>
59 59 #include <sys/modctl.h>
60 60 #include <sys/vmsystm.h>
61 61 #include <vm/page.h>
62 62 #include <sys/atomic.h>
63 63 #include <sys/suntpi.h>
64 64 #include <sys/strlog.h>
65 65 #include <sys/promif.h>
66 66 #include <sys/project.h>
67 67 #include <sys/vm.h>
68 68 #include <sys/taskq.h>
69 69 #include <sys/sunddi.h>
70 70 #include <sys/sunldi_impl.h>
71 71 #include <sys/strsun.h>
72 72 #include <sys/isa_defs.h>
73 73 #include <sys/multidata.h>
74 74 #include <sys/pattr.h>
75 75 #include <sys/strft.h>
76 76 #include <sys/fs/snode.h>
77 77 #include <sys/zone.h>
78 78 #include <sys/open.h>
79 79 #include <sys/sunldi.h>
80 80 #include <sys/sad.h>
81 81 #include <sys/netstack.h>
82 82
83 83 #define O_SAMESTR(q) (((q)->q_next) && \
84 84 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
85 85
86 86 /*
87 87 * WARNING:
88 88 * The variables and routines in this file are private, belonging
89 89 * to the STREAMS subsystem. These should not be used by modules
90 90 * or drivers. Compatibility will not be guaranteed.
91 91 */
92 92
93 93 /*
94 94 * Id value used to distinguish between different multiplexor links.
95 95 */
96 96 static int32_t lnk_id = 0;
97 97
98 98 #define STREAMS_LOPRI MINCLSYSPRI
99 99 static pri_t streams_lopri = STREAMS_LOPRI;
100 100
101 101 #define STRSTAT(x) (str_statistics.x.value.ui64++)
102 102 typedef struct str_stat {
103 103 kstat_named_t sqenables;
104 104 kstat_named_t stenables;
105 105 kstat_named_t syncqservice;
106 106 kstat_named_t freebs;
107 107 kstat_named_t qwr_outer;
108 108 kstat_named_t rservice;
109 109 kstat_named_t strwaits;
110 110 kstat_named_t taskqfails;
111 111 kstat_named_t bufcalls;
112 112 kstat_named_t qhelps;
113 113 kstat_named_t qremoved;
114 114 kstat_named_t sqremoved;
115 115 kstat_named_t bcwaits;
116 116 kstat_named_t sqtoomany;
117 117 } str_stat_t;
118 118
119 119 static str_stat_t str_statistics = {
120 120 { "sqenables", KSTAT_DATA_UINT64 },
121 121 { "stenables", KSTAT_DATA_UINT64 },
122 122 { "syncqservice", KSTAT_DATA_UINT64 },
123 123 { "freebs", KSTAT_DATA_UINT64 },
124 124 { "qwr_outer", KSTAT_DATA_UINT64 },
125 125 { "rservice", KSTAT_DATA_UINT64 },
126 126 { "strwaits", KSTAT_DATA_UINT64 },
127 127 { "taskqfails", KSTAT_DATA_UINT64 },
128 128 { "bufcalls", KSTAT_DATA_UINT64 },
129 129 { "qhelps", KSTAT_DATA_UINT64 },
130 130 { "qremoved", KSTAT_DATA_UINT64 },
131 131 { "sqremoved", KSTAT_DATA_UINT64 },
132 132 { "bcwaits", KSTAT_DATA_UINT64 },
133 133 { "sqtoomany", KSTAT_DATA_UINT64 },
134 134 };
135 135
136 136 static kstat_t *str_kstat;
137 137
138 138 /*
139 139 * qrunflag was used previously to control background scheduling of queues. It
140 140 * is not used anymore, but kept here in case some module still wants to access
141 141 * it via qready() and setqsched macros.
142 142 */
143 143 char qrunflag; /* Unused */
144 144
145 145 /*
146 146 * Most of the streams scheduling is done via task queues. Task queues may fail
147 147 * for non-sleep dispatches, so there are two backup threads servicing failed
148 148 * requests for queues and syncqs. Both of these threads also service failed
149 149 * dispatches freebs requests. Queues are put in the list specified by `qhead'
150 150 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
151 151 * requests are put into `freebs_list' which has no tail pointer. All three
152 152 * lists are protected by a single `service_queue' lock and use
153 153 * `services_to_run' condition variable for signaling background threads. Use of
154 154 * a single lock should not be a problem because it is only used under heavy
155 155 * loads when task queues start to fail and at that time it may be a good idea
156 156 * to throttle scheduling requests.
157 157 *
158 158 * NOTE: queues and syncqs should be scheduled by two separate threads because
159 159 * queue servicing may be blocked waiting for a syncq which may be also
160 160 * scheduled for background execution. This may create a deadlock when only one
161 161 * thread is used for both.
162 162 */
163 163
164 164 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */
165 165
166 166 static kmutex_t service_queue; /* protects all of servicing vars */
167 167 static kcondvar_t services_to_run; /* wake up background service thread */
168 168 static kcondvar_t syncqs_to_run; /* wake up background service thread */
169 169
170 170 /*
171 171 * List of queues scheduled for background processing due to lack of resources
172 172 * in the task queues. Protected by service_queue lock;
173 173 */
174 174 static struct queue *qhead;
175 175 static struct queue *qtail;
176 176
177 177 /*
178 178 * Same list for syncqs
179 179 */
180 180 static syncq_t *sqhead;
181 181 static syncq_t *sqtail;
182 182
183 183 static mblk_t *freebs_list; /* list of buffers to free */
184 184
185 185 /*
186 186 * Backup threads for servicing queues and syncqs
187 187 */
188 188 kthread_t *streams_qbkgrnd_thread;
189 189 kthread_t *streams_sqbkgrnd_thread;
190 190
191 191 /*
192 192 * Bufcalls related variables.
193 193 */
194 194 struct bclist strbcalls; /* list of waiting bufcalls */
195 195 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */
196 196 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */
197 197 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */
198 198 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */
199 199 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */
200 200
201 201 kmutex_t strresources; /* protects global resources */
202 202 kmutex_t muxifier; /* single-threads multiplexor creation */
203 203
204 204 static void *str_stack_init(netstackid_t stackid, netstack_t *ns);
205 205 static void str_stack_shutdown(netstackid_t stackid, void *arg);
206 206 static void str_stack_fini(netstackid_t stackid, void *arg);
207 207
208 208 /*
209 209 * run_queues is no longer used, but is kept in case some 3rd party
210 210 * module/driver decides to use it.
211 211 */
212 212 int run_queues = 0;
213 213
214 214 /*
215 215 * sq_max_size is the depth of the syncq (in number of messages) before
216 216 * qfill_syncq() starts QFULL'ing destination queues. As its primary
217 217 * consumer - IP is no longer D_MTPERMOD, but there may be other
218 218 * modules/drivers depend on this syncq flow control, we prefer to
219 219 * choose a large number as the default value. For potential
220 220 * performance gain, this value is tunable in /etc/system.
221 221 */
222 222 int sq_max_size = 10000;
223 223
224 224 /*
225 225 * The number of ciputctrl structures per syncq and stream we create when
226 226 * needed.
227 227 */
228 228 int n_ciputctrl;
229 229 int max_n_ciputctrl = 16;
230 230 /*
231 231 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
232 232 */
233 233 int min_n_ciputctrl = 2;
234 234
235 235 /*
236 236 * Per-driver/module syncqs
237 237 * ========================
238 238 *
239 239 * For drivers/modules that use PERMOD or outer syncqs we keep a list of
240 240 * perdm structures, new entries being added (and new syncqs allocated) when
241 241 * setq() encounters a module/driver with a streamtab that it hasn't seen
242 242 * before.
243 243 * The reason for this mechanism is that some modules and drivers share a
244 244 * common streamtab and it is necessary for those modules and drivers to also
245 245 * share a common PERMOD syncq.
246 246 *
247 247 * perdm_list --> dm_str == streamtab_1
248 248 * dm_sq == syncq_1
249 249 * dm_ref
250 250 * dm_next --> dm_str == streamtab_2
251 251 * dm_sq == syncq_2
252 252 * dm_ref
253 253 * dm_next --> ... NULL
254 254 *
255 255 * The dm_ref field is incremented for each new driver/module that takes
256 256 * a reference to the perdm structure and hence shares the syncq.
257 257 * References are held in the fmodsw_impl_t structure for each STREAMS module
258 258 * or the dev_impl array (indexed by device major number) for each driver.
259 259 *
260 260 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
261 261 * ^ ^ ^ ^
262 262 * | ______________/ | |
263 263 * | / | |
264 264 * dev_impl: ...|x|y|... module A module B
265 265 *
266 266 * When a module/driver is unloaded the reference count is decremented and,
267 267 * when it falls to zero, the perdm structure is removed from the list and
268 268 * the syncq is freed (see rele_dm()).
269 269 */
270 270 perdm_t *perdm_list = NULL;
271 271 static krwlock_t perdm_rwlock;
272 272 cdevsw_impl_t *devimpl;
273 273
274 274 extern struct qinit strdata;
275 275 extern struct qinit stwdata;
276 276
277 277 static void runservice(queue_t *);
278 278 static void streams_bufcall_service(void);
279 279 static void streams_qbkgrnd_service(void);
280 280 static void streams_sqbkgrnd_service(void);
281 281 static syncq_t *new_syncq(void);
282 282 static void free_syncq(syncq_t *);
283 283 static void outer_insert(syncq_t *, syncq_t *);
284 284 static void outer_remove(syncq_t *, syncq_t *);
285 285 static void write_now(syncq_t *);
286 286 static void clr_qfull(queue_t *);
287 287 static void runbufcalls(void);
288 288 static void sqenable(syncq_t *);
289 289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
290 290 static void wait_q_syncq(queue_t *);
291 291 static void backenable_insertedq(queue_t *);
292 292
293 293 static void queue_service(queue_t *);
294 294 static void stream_service(stdata_t *);
295 295 static void syncq_service(syncq_t *);
296 296 static void qwriter_outer_service(syncq_t *);
297 297 static void mblk_free(mblk_t *);
298 298 #ifdef DEBUG
299 299 static int qprocsareon(queue_t *);
300 300 #endif
301 301
302 302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
303 303 static void reset_nfsrv_ptr(queue_t *, queue_t *);
304 304 void set_qfull(queue_t *);
305 305
306 306 static void sq_run_events(syncq_t *);
307 307 static int propagate_syncq(queue_t *);
308 308
309 309 static void blocksq(syncq_t *, ushort_t, int);
310 310 static void unblocksq(syncq_t *, ushort_t, int);
311 311 static int dropsq(syncq_t *, uint16_t);
312 312 static void emptysq(syncq_t *);
313 313 static sqlist_t *sqlist_alloc(struct stdata *, int);
314 314 static void sqlist_free(sqlist_t *);
315 315 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t);
316 316 static void sqlist_insert(sqlist_t *, syncq_t *);
317 317 static void sqlist_insertall(sqlist_t *, queue_t *);
318 318
319 319 static void strsetuio(stdata_t *);
320 320
321 321 struct kmem_cache *stream_head_cache;
322 322 struct kmem_cache *queue_cache;
323 323 struct kmem_cache *syncq_cache;
324 324 struct kmem_cache *qband_cache;
325 325 struct kmem_cache *linkinfo_cache;
326 326 struct kmem_cache *ciputctrl_cache = NULL;
327 327
328 328 static linkinfo_t *linkinfo_list;
329 329
330 330 /* Global esballoc throttling queue */
331 331 static esb_queue_t system_esbq;
332 332
333 333 /* Array of esballoc throttling queues, of length esbq_nelem */
334 334 static esb_queue_t *volatile system_esbq_array;
335 335 static int esbq_nelem;
336 336 static kmutex_t esbq_lock;
337 337 static int esbq_log2_cpus_per_q = 0;
338 338
339 339 /* Scale the system_esbq length by setting number of CPUs per queue. */
340 340 uint_t esbq_cpus_per_q = 1;
341 341
342 342 /*
343 343 * esballoc tunable parameters.
344 344 */
345 345 int esbq_max_qlen = 0x16; /* throttled queue length */
346 346 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */
347 347
348 348 /*
349 349 * Routines to handle esballoc queueing.
350 350 */
351 351 static void esballoc_process_queue(esb_queue_t *);
352 352 static void esballoc_enqueue_mblk(mblk_t *);
353 353 static void esballoc_timer(void *);
354 354 static void esballoc_set_timer(esb_queue_t *, clock_t);
355 355 static void esballoc_mblk_free(mblk_t *);
356 356
357 357 /*
358 358 * Qinit structure and Module_info structures
359 359 * for passthru read and write queues
360 360 */
361 361
362 362 static void pass_wput(queue_t *, mblk_t *);
363 363 static queue_t *link_addpassthru(stdata_t *);
364 364 static void link_rempassthru(queue_t *);
365 365
366 366 struct module_info passthru_info = {
367 367 0,
368 368 "passthru",
369 369 0,
370 370 INFPSZ,
371 371 STRHIGH,
372 372 STRLOW
373 373 };
374 374
375 375 struct qinit passthru_rinit = {
376 376 (int (*)())putnext,
377 377 NULL,
378 378 NULL,
379 379 NULL,
380 380 NULL,
381 381 &passthru_info,
382 382 NULL
383 383 };
384 384
385 385 struct qinit passthru_winit = {
386 386 (int (*)()) pass_wput,
387 387 NULL,
388 388 NULL,
389 389 NULL,
390 390 NULL,
391 391 &passthru_info,
392 392 NULL
393 393 };
394 394
395 395 /*
396 396 * Verify correctness of list head/tail pointers.
397 397 */
398 398 #define LISTCHECK(head, tail, link) { \
399 399 EQUIV(head, tail); \
400 400 IMPLY(tail != NULL, tail->link == NULL); \
401 401 }
402 402
403 403 /*
404 404 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
405 405 * using a `link' field.
406 406 */
407 407 #define ENQUEUE(el, head, tail, link) { \
408 408 ASSERT(el->link == NULL); \
409 409 LISTCHECK(head, tail, link); \
410 410 if (head == NULL) \
411 411 head = el; \
412 412 else \
413 413 tail->link = el; \
414 414 tail = el; \
415 415 }
416 416
417 417 /*
418 418 * Dequeue the first element of the list denoted by `head' and `tail' pointers
419 419 * using a `link' field and put result into `el'.
420 420 */
421 421 #define DQ(el, head, tail, link) { \
422 422 LISTCHECK(head, tail, link); \
423 423 el = head; \
424 424 if (head != NULL) { \
425 425 head = head->link; \
426 426 if (head == NULL) \
427 427 tail = NULL; \
428 428 el->link = NULL; \
429 429 } \
430 430 }
431 431
432 432 /*
433 433 * Remove `el' from the list using `chase' and `curr' pointers and return result
434 434 * in `succeed'.
435 435 */
436 436 #define RMQ(el, head, tail, link, chase, curr, succeed) { \
437 437 LISTCHECK(head, tail, link); \
438 438 chase = NULL; \
439 439 succeed = 0; \
440 440 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
441 441 chase = curr; \
442 442 if (curr != NULL) { \
443 443 succeed = 1; \
444 444 ASSERT(curr == el); \
445 445 if (chase != NULL) \
446 446 chase->link = curr->link; \
447 447 else \
448 448 head = curr->link; \
449 449 curr->link = NULL; \
450 450 if (curr == tail) \
451 451 tail = chase; \
452 452 } \
453 453 LISTCHECK(head, tail, link); \
454 454 }
455 455
456 456 /* Handling of delayed messages on the inner syncq. */
457 457
458 458 /*
459 459 * DEBUG versions should use function versions (to simplify tracing) and
460 460 * non-DEBUG kernels should use macro versions.
461 461 */
462 462
463 463 /*
464 464 * Put a queue on the syncq list of queues.
465 465 * Assumes SQLOCK held.
466 466 */
467 467 #define SQPUT_Q(sq, qp) \
468 468 { \
469 469 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
470 470 if (!(qp->q_sqflags & Q_SQQUEUED)) { \
471 471 /* The queue should not be linked anywhere */ \
472 472 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
473 473 /* Head and tail may only be NULL simultaneously */ \
474 474 EQUIV(sq->sq_head, sq->sq_tail); \
475 475 /* Queue may be only enqueued on its syncq */ \
476 476 ASSERT(sq == qp->q_syncq); \
477 477 /* Check the correctness of SQ_MESSAGES flag */ \
478 478 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
479 479 /* Sanity check first/last elements of the list */ \
480 480 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
481 481 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
482 482 /* \
483 483 * Sanity check of priority field: empty queue should \
484 484 * have zero priority \
485 485 * and nqueues equal to zero. \
486 486 */ \
487 487 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
488 488 /* Sanity check of sq_nqueues field */ \
489 489 EQUIV(sq->sq_head, sq->sq_nqueues); \
490 490 if (sq->sq_head == NULL) { \
491 491 sq->sq_head = sq->sq_tail = qp; \
492 492 sq->sq_flags |= SQ_MESSAGES; \
493 493 } else if (qp->q_spri == 0) { \
494 494 qp->q_sqprev = sq->sq_tail; \
495 495 sq->sq_tail->q_sqnext = qp; \
496 496 sq->sq_tail = qp; \
497 497 } else { \
498 498 /* \
499 499 * Put this queue in priority order: higher \
500 500 * priority gets closer to the head. \
501 501 */ \
502 502 queue_t **qpp = &sq->sq_tail; \
503 503 queue_t *qnext = NULL; \
504 504 \
505 505 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
506 506 qnext = *qpp; \
507 507 qpp = &(*qpp)->q_sqprev; \
508 508 } \
509 509 qp->q_sqnext = qnext; \
510 510 qp->q_sqprev = *qpp; \
511 511 if (*qpp != NULL) { \
512 512 (*qpp)->q_sqnext = qp; \
513 513 } else { \
514 514 sq->sq_head = qp; \
515 515 sq->sq_pri = sq->sq_head->q_spri; \
516 516 } \
517 517 *qpp = qp; \
518 518 } \
519 519 qp->q_sqflags |= Q_SQQUEUED; \
520 520 qp->q_sqtstamp = ddi_get_lbolt(); \
521 521 sq->sq_nqueues++; \
522 522 } \
523 523 }
524 524
525 525 /*
526 526 * Remove a queue from the syncq list
527 527 * Assumes SQLOCK held.
528 528 */
529 529 #define SQRM_Q(sq, qp) \
530 530 { \
531 531 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
532 532 ASSERT(qp->q_sqflags & Q_SQQUEUED); \
533 533 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
534 534 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
535 535 /* Check that the queue is actually in the list */ \
536 536 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
537 537 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
538 538 ASSERT(sq->sq_nqueues != 0); \
539 539 if (qp->q_sqprev == NULL) { \
540 540 /* First queue on list, make head q_sqnext */ \
541 541 sq->sq_head = qp->q_sqnext; \
542 542 } else { \
543 543 /* Make prev->next == next */ \
544 544 qp->q_sqprev->q_sqnext = qp->q_sqnext; \
545 545 } \
546 546 if (qp->q_sqnext == NULL) { \
547 547 /* Last queue on list, make tail sqprev */ \
548 548 sq->sq_tail = qp->q_sqprev; \
549 549 } else { \
550 550 /* Make next->prev == prev */ \
551 551 qp->q_sqnext->q_sqprev = qp->q_sqprev; \
552 552 } \
553 553 /* clear out references on this queue */ \
554 554 qp->q_sqprev = qp->q_sqnext = NULL; \
555 555 qp->q_sqflags &= ~Q_SQQUEUED; \
556 556 /* If there is nothing queued, clear SQ_MESSAGES */ \
557 557 if (sq->sq_head != NULL) { \
558 558 sq->sq_pri = sq->sq_head->q_spri; \
559 559 } else { \
560 560 sq->sq_flags &= ~SQ_MESSAGES; \
561 561 sq->sq_pri = 0; \
562 562 } \
563 563 sq->sq_nqueues--; \
564 564 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
565 565 (sq->sq_flags & SQ_QUEUED) == 0); \
566 566 }
567 567
568 568 /* Hide the definition from the header file. */
569 569 #ifdef SQPUT_MP
570 570 #undef SQPUT_MP
571 571 #endif
572 572
573 573 /*
574 574 * Put a message on the queue syncq.
575 575 * Assumes QLOCK held.
576 576 */
577 577 #define SQPUT_MP(qp, mp) \
578 578 { \
579 579 ASSERT(MUTEX_HELD(QLOCK(qp))); \
580 580 ASSERT(qp->q_sqhead == NULL || \
581 581 (qp->q_sqtail != NULL && \
582 582 qp->q_sqtail->b_next == NULL)); \
583 583 qp->q_syncqmsgs++; \
584 584 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \
585 585 if (qp->q_sqhead == NULL) { \
586 586 qp->q_sqhead = qp->q_sqtail = mp; \
587 587 } else { \
588 588 qp->q_sqtail->b_next = mp; \
589 589 qp->q_sqtail = mp; \
590 590 } \
591 591 ASSERT(qp->q_syncqmsgs > 0); \
592 592 set_qfull(qp); \
593 593 }
594 594
595 595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \
596 596 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
597 597 if ((sq)->sq_ciputctrl != NULL) { \
598 598 int i; \
599 599 int nlocks = (sq)->sq_nciputctrl; \
600 600 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
601 601 ASSERT((sq)->sq_type & SQ_CIPUT); \
602 602 for (i = 0; i <= nlocks; i++) { \
603 603 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
604 604 cip[i].ciputctrl_count |= SQ_FASTPUT; \
605 605 } \
606 606 } \
607 607 }
608 608
609 609
610 610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \
611 611 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
612 612 if ((sq)->sq_ciputctrl != NULL) { \
613 613 int i; \
614 614 int nlocks = (sq)->sq_nciputctrl; \
615 615 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
616 616 ASSERT((sq)->sq_type & SQ_CIPUT); \
617 617 for (i = 0; i <= nlocks; i++) { \
618 618 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
619 619 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \
620 620 } \
621 621 } \
622 622 }
623 623
624 624 /*
625 625 * Run service procedures for all queues in the stream head.
626 626 */
627 627 #define STR_SERVICE(stp, q) { \
628 628 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \
629 629 while (stp->sd_qhead != NULL) { \
630 630 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \
631 631 ASSERT(stp->sd_nqueues > 0); \
632 632 stp->sd_nqueues--; \
633 633 ASSERT(!(q->q_flag & QINSERVICE)); \
634 634 mutex_exit(&stp->sd_qlock); \
635 635 queue_service(q); \
636 636 mutex_enter(&stp->sd_qlock); \
637 637 } \
638 638 ASSERT(stp->sd_nqueues == 0); \
639 639 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \
640 640 }
641 641
642 642 /*
643 643 * Constructor/destructor routines for the stream head cache
↓ open down ↓ |
643 lines elided |
↑ open up ↑ |
644 644 */
645 645 /* ARGSUSED */
646 646 static int
647 647 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
648 648 {
649 649 stdata_t *stp = buf;
650 650
651 651 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
652 652 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
653 653 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
654 + mutex_init(&stp->sd_pid_tree_lock, NULL, MUTEX_DEFAULT, NULL);
654 655 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
655 656 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
656 657 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
657 658 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
658 659 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
660 + avl_create(&stp->sd_pid_tree, pid_node_comparator, sizeof (pid_node_t),
661 + offsetof(pid_node_t, pn_ref_link));
659 662 stp->sd_wrq = NULL;
660 663
661 664 return (0);
662 665 }
663 666
664 667 /* ARGSUSED */
665 668 static void
666 669 stream_head_destructor(void *buf, void *cdrarg)
667 670 {
668 671 stdata_t *stp = buf;
669 672
670 673 mutex_destroy(&stp->sd_lock);
671 674 mutex_destroy(&stp->sd_reflock);
672 675 mutex_destroy(&stp->sd_qlock);
676 + mutex_destroy(&stp->sd_pid_tree_lock);
673 677 cv_destroy(&stp->sd_monitor);
674 678 cv_destroy(&stp->sd_iocmonitor);
675 679 cv_destroy(&stp->sd_refmonitor);
676 680 cv_destroy(&stp->sd_qcv);
677 681 cv_destroy(&stp->sd_zcopy_wait);
682 + avl_destroy(&stp->sd_pid_tree);
678 683 }
679 684
680 685 /*
681 686 * Constructor/destructor routines for the queue cache
682 687 */
683 688 /* ARGSUSED */
684 689 static int
685 690 queue_constructor(void *buf, void *cdrarg, int kmflags)
686 691 {
687 692 queinfo_t *qip = buf;
688 693 queue_t *qp = &qip->qu_rqueue;
689 694 queue_t *wqp = &qip->qu_wqueue;
690 695 syncq_t *sq = &qip->qu_syncq;
691 696
692 697 qp->q_first = NULL;
693 698 qp->q_link = NULL;
694 699 qp->q_count = 0;
695 700 qp->q_mblkcnt = 0;
696 701 qp->q_sqhead = NULL;
697 702 qp->q_sqtail = NULL;
698 703 qp->q_sqnext = NULL;
699 704 qp->q_sqprev = NULL;
700 705 qp->q_sqflags = 0;
701 706 qp->q_rwcnt = 0;
702 707 qp->q_spri = 0;
703 708
704 709 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
705 710 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
706 711
707 712 wqp->q_first = NULL;
708 713 wqp->q_link = NULL;
709 714 wqp->q_count = 0;
710 715 wqp->q_mblkcnt = 0;
711 716 wqp->q_sqhead = NULL;
712 717 wqp->q_sqtail = NULL;
713 718 wqp->q_sqnext = NULL;
714 719 wqp->q_sqprev = NULL;
715 720 wqp->q_sqflags = 0;
716 721 wqp->q_rwcnt = 0;
717 722 wqp->q_spri = 0;
718 723
719 724 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
720 725 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
721 726
722 727 sq->sq_head = NULL;
723 728 sq->sq_tail = NULL;
724 729 sq->sq_evhead = NULL;
725 730 sq->sq_evtail = NULL;
726 731 sq->sq_callbpend = NULL;
727 732 sq->sq_outer = NULL;
728 733 sq->sq_onext = NULL;
729 734 sq->sq_oprev = NULL;
730 735 sq->sq_next = NULL;
731 736 sq->sq_svcflags = 0;
732 737 sq->sq_servcount = 0;
733 738 sq->sq_needexcl = 0;
734 739 sq->sq_nqueues = 0;
735 740 sq->sq_pri = 0;
736 741
737 742 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
738 743 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
739 744 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
740 745
741 746 return (0);
742 747 }
743 748
744 749 /* ARGSUSED */
745 750 static void
746 751 queue_destructor(void *buf, void *cdrarg)
747 752 {
748 753 queinfo_t *qip = buf;
749 754 queue_t *qp = &qip->qu_rqueue;
750 755 queue_t *wqp = &qip->qu_wqueue;
751 756 syncq_t *sq = &qip->qu_syncq;
752 757
753 758 ASSERT(qp->q_sqhead == NULL);
754 759 ASSERT(wqp->q_sqhead == NULL);
755 760 ASSERT(qp->q_sqnext == NULL);
756 761 ASSERT(wqp->q_sqnext == NULL);
757 762 ASSERT(qp->q_rwcnt == 0);
758 763 ASSERT(wqp->q_rwcnt == 0);
759 764
760 765 mutex_destroy(&qp->q_lock);
761 766 cv_destroy(&qp->q_wait);
762 767
763 768 mutex_destroy(&wqp->q_lock);
764 769 cv_destroy(&wqp->q_wait);
765 770
766 771 mutex_destroy(&sq->sq_lock);
767 772 cv_destroy(&sq->sq_wait);
768 773 cv_destroy(&sq->sq_exitwait);
769 774 }
770 775
771 776 /*
772 777 * Constructor/destructor routines for the syncq cache
773 778 */
774 779 /* ARGSUSED */
775 780 static int
776 781 syncq_constructor(void *buf, void *cdrarg, int kmflags)
777 782 {
778 783 syncq_t *sq = buf;
779 784
780 785 bzero(buf, sizeof (syncq_t));
781 786
782 787 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
783 788 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
784 789 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
785 790
786 791 return (0);
787 792 }
788 793
789 794 /* ARGSUSED */
790 795 static void
791 796 syncq_destructor(void *buf, void *cdrarg)
792 797 {
793 798 syncq_t *sq = buf;
794 799
795 800 ASSERT(sq->sq_head == NULL);
796 801 ASSERT(sq->sq_tail == NULL);
797 802 ASSERT(sq->sq_evhead == NULL);
798 803 ASSERT(sq->sq_evtail == NULL);
799 804 ASSERT(sq->sq_callbpend == NULL);
800 805 ASSERT(sq->sq_callbflags == 0);
801 806 ASSERT(sq->sq_outer == NULL);
802 807 ASSERT(sq->sq_onext == NULL);
803 808 ASSERT(sq->sq_oprev == NULL);
804 809 ASSERT(sq->sq_next == NULL);
805 810 ASSERT(sq->sq_needexcl == 0);
806 811 ASSERT(sq->sq_svcflags == 0);
807 812 ASSERT(sq->sq_servcount == 0);
808 813 ASSERT(sq->sq_nqueues == 0);
809 814 ASSERT(sq->sq_pri == 0);
810 815 ASSERT(sq->sq_count == 0);
811 816 ASSERT(sq->sq_rmqcount == 0);
812 817 ASSERT(sq->sq_cancelid == 0);
813 818 ASSERT(sq->sq_ciputctrl == NULL);
814 819 ASSERT(sq->sq_nciputctrl == 0);
815 820 ASSERT(sq->sq_type == 0);
816 821 ASSERT(sq->sq_flags == 0);
817 822
818 823 mutex_destroy(&sq->sq_lock);
819 824 cv_destroy(&sq->sq_wait);
820 825 cv_destroy(&sq->sq_exitwait);
821 826 }
822 827
823 828 /* ARGSUSED */
824 829 static int
825 830 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
826 831 {
827 832 ciputctrl_t *cip = buf;
828 833 int i;
829 834
830 835 for (i = 0; i < n_ciputctrl; i++) {
831 836 cip[i].ciputctrl_count = SQ_FASTPUT;
832 837 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
833 838 }
834 839
835 840 return (0);
836 841 }
837 842
838 843 /* ARGSUSED */
839 844 static void
840 845 ciputctrl_destructor(void *buf, void *cdrarg)
841 846 {
842 847 ciputctrl_t *cip = buf;
843 848 int i;
844 849
845 850 for (i = 0; i < n_ciputctrl; i++) {
846 851 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
847 852 mutex_destroy(&cip[i].ciputctrl_lock);
848 853 }
849 854 }
850 855
851 856 /*
852 857 * Init routine run from main at boot time.
853 858 */
854 859 void
855 860 strinit(void)
856 861 {
857 862 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
858 863
859 864 stream_head_cache = kmem_cache_create("stream_head_cache",
860 865 sizeof (stdata_t), 0,
861 866 stream_head_constructor, stream_head_destructor, NULL,
862 867 NULL, NULL, 0);
863 868
864 869 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
865 870 queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
866 871
867 872 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
868 873 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
869 874
870 875 qband_cache = kmem_cache_create("qband_cache",
871 876 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
872 877
873 878 linkinfo_cache = kmem_cache_create("linkinfo_cache",
874 879 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
875 880
876 881 n_ciputctrl = ncpus;
877 882 n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
878 883 ASSERT(n_ciputctrl >= 1);
879 884 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
880 885 if (n_ciputctrl >= min_n_ciputctrl) {
881 886 ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
882 887 sizeof (ciputctrl_t) * n_ciputctrl,
883 888 sizeof (ciputctrl_t), ciputctrl_constructor,
884 889 ciputctrl_destructor, NULL, NULL, NULL, 0);
885 890 }
886 891
887 892 streams_taskq = system_taskq;
888 893
889 894 if (streams_taskq == NULL)
890 895 panic("strinit: no memory for streams taskq!");
891 896
892 897 bc_bkgrnd_thread = thread_create(NULL, 0,
893 898 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
894 899
895 900 streams_qbkgrnd_thread = thread_create(NULL, 0,
896 901 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
897 902
898 903 streams_sqbkgrnd_thread = thread_create(NULL, 0,
899 904 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
900 905
901 906 /*
902 907 * Create STREAMS kstats.
903 908 */
904 909 str_kstat = kstat_create("streams", 0, "strstat",
905 910 "net", KSTAT_TYPE_NAMED,
906 911 sizeof (str_statistics) / sizeof (kstat_named_t),
907 912 KSTAT_FLAG_VIRTUAL);
908 913
909 914 if (str_kstat != NULL) {
910 915 str_kstat->ks_data = &str_statistics;
911 916 kstat_install(str_kstat);
912 917 }
913 918
914 919 /*
915 920 * TPI support routine initialisation.
916 921 */
917 922 tpi_init();
918 923
919 924 /*
920 925 * Handle to have autopush and persistent link information per
921 926 * zone.
922 927 * Note: uses shutdown hook instead of destroy hook so that the
923 928 * persistent links can be torn down before the destroy hooks
924 929 * in the TCP/IP stack are called.
925 930 */
926 931 netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
927 932 str_stack_fini);
928 933 }
929 934
930 935 void
931 936 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
932 937 {
933 938 struct stdata *stp;
934 939
935 940 ASSERT(vp->v_stream);
936 941 stp = vp->v_stream;
937 942 /* Have to hold sd_lock to prevent siglist from changing */
938 943 mutex_enter(&stp->sd_lock);
939 944 if (stp->sd_sigflags & event)
940 945 strsendsig(stp->sd_siglist, event, band, error);
941 946 mutex_exit(&stp->sd_lock);
↓ open down ↓ |
254 lines elided |
↑ open up ↑ |
942 947 }
943 948
944 949 /*
945 950 * Send the "sevent" set of signals to a process.
946 951 * This might send more than one signal if the process is registered
947 952 * for multiple events. The caller should pass in an sevent that only
948 953 * includes the events for which the process has registered.
949 954 */
950 955 static void
951 956 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
952 - uchar_t band, int error)
957 + uchar_t band, int error)
953 958 {
954 959 ASSERT(MUTEX_HELD(&proc->p_lock));
955 960
956 961 info->si_band = 0;
957 962 info->si_errno = 0;
958 963
959 964 if (sevent & S_ERROR) {
960 965 sevent &= ~S_ERROR;
961 966 info->si_code = POLL_ERR;
962 967 info->si_errno = error;
963 968 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
964 969 "strsendsig:proc %p info %p", proc, info);
965 970 sigaddq(proc, NULL, info, KM_NOSLEEP);
966 971 info->si_errno = 0;
967 972 }
968 973 if (sevent & S_HANGUP) {
969 974 sevent &= ~S_HANGUP;
970 975 info->si_code = POLL_HUP;
971 976 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
972 977 "strsendsig:proc %p info %p", proc, info);
973 978 sigaddq(proc, NULL, info, KM_NOSLEEP);
974 979 }
975 980 if (sevent & S_HIPRI) {
976 981 sevent &= ~S_HIPRI;
977 982 info->si_code = POLL_PRI;
978 983 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
979 984 "strsendsig:proc %p info %p", proc, info);
980 985 sigaddq(proc, NULL, info, KM_NOSLEEP);
981 986 }
982 987 if (sevent & S_RDBAND) {
983 988 sevent &= ~S_RDBAND;
984 989 if (events & S_BANDURG)
985 990 sigtoproc(proc, NULL, SIGURG);
986 991 else
987 992 sigtoproc(proc, NULL, SIGPOLL);
988 993 }
989 994 if (sevent & S_WRBAND) {
990 995 sevent &= ~S_WRBAND;
991 996 sigtoproc(proc, NULL, SIGPOLL);
992 997 }
993 998 if (sevent & S_INPUT) {
994 999 sevent &= ~S_INPUT;
995 1000 info->si_code = POLL_IN;
996 1001 info->si_band = band;
997 1002 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
998 1003 "strsendsig:proc %p info %p", proc, info);
999 1004 sigaddq(proc, NULL, info, KM_NOSLEEP);
1000 1005 info->si_band = 0;
1001 1006 }
1002 1007 if (sevent & S_OUTPUT) {
1003 1008 sevent &= ~S_OUTPUT;
1004 1009 info->si_code = POLL_OUT;
1005 1010 info->si_band = band;
1006 1011 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1007 1012 "strsendsig:proc %p info %p", proc, info);
1008 1013 sigaddq(proc, NULL, info, KM_NOSLEEP);
1009 1014 info->si_band = 0;
1010 1015 }
1011 1016 if (sevent & S_MSG) {
1012 1017 sevent &= ~S_MSG;
1013 1018 info->si_code = POLL_MSG;
1014 1019 info->si_band = band;
1015 1020 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1016 1021 "strsendsig:proc %p info %p", proc, info);
1017 1022 sigaddq(proc, NULL, info, KM_NOSLEEP);
1018 1023 info->si_band = 0;
1019 1024 }
1020 1025 if (sevent & S_RDNORM) {
1021 1026 sevent &= ~S_RDNORM;
1022 1027 sigtoproc(proc, NULL, SIGPOLL);
1023 1028 }
1024 1029 if (sevent != 0) {
1025 1030 panic("strsendsig: unknown event(s) %x", sevent);
1026 1031 }
1027 1032 }
1028 1033
1029 1034 /*
1030 1035 * Send SIGPOLL/SIGURG signal to all processes and process groups
1031 1036 * registered on the given signal list that want a signal for at
1032 1037 * least one of the specified events.
1033 1038 *
1034 1039 * Must be called with exclusive access to siglist (caller holding sd_lock).
1035 1040 *
1036 1041 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1037 1042 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1038 1043 * while it is in the siglist.
1039 1044 *
1040 1045 * For performance reasons (MP scalability) the code drops pidlock
1041 1046 * when sending signals to a single process.
1042 1047 * When sending to a process group the code holds
1043 1048 * pidlock to prevent the membership in the process group from changing
1044 1049 * while walking the p_pglink list.
1045 1050 */
1046 1051 void
1047 1052 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1048 1053 {
1049 1054 strsig_t *ssp;
1050 1055 k_siginfo_t info;
1051 1056 struct pid *pidp;
1052 1057 proc_t *proc;
1053 1058
1054 1059 info.si_signo = SIGPOLL;
1055 1060 info.si_errno = 0;
1056 1061 for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1057 1062 int sevent;
1058 1063
1059 1064 sevent = ssp->ss_events & event;
1060 1065 if (sevent == 0)
1061 1066 continue;
1062 1067
1063 1068 if ((pidp = ssp->ss_pidp) == NULL) {
1064 1069 /* pid was released but still on event list */
1065 1070 continue;
1066 1071 }
1067 1072
1068 1073
1069 1074 if (ssp->ss_pid > 0) {
1070 1075 /*
1071 1076 * XXX This unfortunately still generates
1072 1077 * a signal when a fd is closed but
1073 1078 * the proc is active.
1074 1079 */
1075 1080 ASSERT(ssp->ss_pid == pidp->pid_id);
1076 1081
1077 1082 mutex_enter(&pidlock);
1078 1083 proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1079 1084 if (proc == NULL) {
1080 1085 mutex_exit(&pidlock);
1081 1086 continue;
1082 1087 }
1083 1088 mutex_enter(&proc->p_lock);
1084 1089 mutex_exit(&pidlock);
1085 1090 dosendsig(proc, ssp->ss_events, sevent, &info,
1086 1091 band, error);
1087 1092 mutex_exit(&proc->p_lock);
1088 1093 } else {
1089 1094 /*
1090 1095 * Send to process group. Hold pidlock across
1091 1096 * calls to dosendsig().
1092 1097 */
1093 1098 pid_t pgrp = -ssp->ss_pid;
1094 1099
1095 1100 mutex_enter(&pidlock);
1096 1101 proc = pgfind_zone(pgrp, ALL_ZONES);
1097 1102 while (proc != NULL) {
1098 1103 mutex_enter(&proc->p_lock);
1099 1104 dosendsig(proc, ssp->ss_events, sevent,
1100 1105 &info, band, error);
1101 1106 mutex_exit(&proc->p_lock);
1102 1107 proc = proc->p_pglink;
1103 1108 }
1104 1109 mutex_exit(&pidlock);
1105 1110 }
1106 1111 }
1107 1112 }
1108 1113
1109 1114 /*
1110 1115 * Attach a stream device or module.
1111 1116 * qp is a read queue; the new queue goes in so its next
1112 1117 * read ptr is the argument, and the write queue corresponding
1113 1118 * to the argument points to this queue. Return 0 on success,
1114 1119 * or a non-zero errno on failure.
1115 1120 */
1116 1121 int
1117 1122 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1118 1123 boolean_t is_insert)
1119 1124 {
1120 1125 major_t major;
1121 1126 cdevsw_impl_t *dp;
1122 1127 struct streamtab *str;
1123 1128 queue_t *rq;
1124 1129 queue_t *wrq;
1125 1130 uint32_t qflag;
1126 1131 uint32_t sqtype;
1127 1132 perdm_t *dmp;
1128 1133 int error;
1129 1134 int sflag;
1130 1135
1131 1136 rq = allocq();
1132 1137 wrq = _WR(rq);
1133 1138 STREAM(rq) = STREAM(wrq) = STREAM(qp);
1134 1139
1135 1140 if (fp != NULL) {
1136 1141 str = fp->f_str;
1137 1142 qflag = fp->f_qflag;
1138 1143 sqtype = fp->f_sqtype;
1139 1144 dmp = fp->f_dmp;
1140 1145 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1141 1146 sflag = MODOPEN;
1142 1147
1143 1148 /*
1144 1149 * stash away a pointer to the module structure so we can
1145 1150 * unref it in qdetach.
1146 1151 */
1147 1152 rq->q_fp = fp;
1148 1153 } else {
1149 1154 ASSERT(!is_insert);
1150 1155
1151 1156 major = getmajor(*devp);
1152 1157 dp = &devimpl[major];
1153 1158
1154 1159 str = dp->d_str;
1155 1160 ASSERT(str == STREAMSTAB(major));
1156 1161
1157 1162 qflag = dp->d_qflag;
1158 1163 ASSERT(qflag & QISDRV);
1159 1164 sqtype = dp->d_sqtype;
1160 1165
1161 1166 /* create perdm_t if needed */
1162 1167 if (NEED_DM(dp->d_dmp, qflag))
1163 1168 dp->d_dmp = hold_dm(str, qflag, sqtype);
1164 1169
1165 1170 dmp = dp->d_dmp;
1166 1171 sflag = 0;
1167 1172 }
1168 1173
1169 1174 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1170 1175 "qattach:qflag == %X(%X)", qflag, *devp);
1171 1176
1172 1177 /* setq might sleep in allocator - avoid holding locks. */
1173 1178 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1174 1179
1175 1180 /*
1176 1181 * Before calling the module's open routine, set up the q_next
1177 1182 * pointer for inserting a module in the middle of a stream.
1178 1183 *
1179 1184 * Note that we can always set _QINSERTING and set up q_next
1180 1185 * pointer for both inserting and pushing a module. Then there
1181 1186 * is no need for the is_insert parameter. In insertq(), called
1182 1187 * by qprocson(), assume that q_next of the new module always points
1183 1188 * to the correct queue and use it for insertion. Everything should
1184 1189 * work out fine. But in the first release of _I_INSERT, we
1185 1190 * distinguish between inserting and pushing to make sure that
1186 1191 * pushing a module follows the same code path as before.
1187 1192 */
1188 1193 if (is_insert) {
1189 1194 rq->q_flag |= _QINSERTING;
1190 1195 rq->q_next = qp;
1191 1196 }
1192 1197
1193 1198 /*
1194 1199 * If there is an outer perimeter get exclusive access during
1195 1200 * the open procedure. Bump up the reference count on the queue.
1196 1201 */
1197 1202 entersq(rq->q_syncq, SQ_OPENCLOSE);
1198 1203 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1199 1204 if (error != 0)
1200 1205 goto failed;
1201 1206 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1202 1207 ASSERT(qprocsareon(rq));
1203 1208 return (0);
1204 1209
1205 1210 failed:
1206 1211 rq->q_flag &= ~_QINSERTING;
1207 1212 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1208 1213 qprocsoff(rq);
1209 1214 leavesq(rq->q_syncq, SQ_OPENCLOSE);
1210 1215 rq->q_next = wrq->q_next = NULL;
1211 1216 qdetach(rq, 0, 0, crp, B_FALSE);
1212 1217 return (error);
1213 1218 }
1214 1219
1215 1220 /*
1216 1221 * Handle second open of stream. For modules, set the
1217 1222 * last argument to MODOPEN and do not pass any open flags.
1218 1223 * Ignore dummydev since this is not the first open.
1219 1224 */
1220 1225 int
1221 1226 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1222 1227 {
1223 1228 int error;
1224 1229 dev_t dummydev;
1225 1230 queue_t *wqp = _WR(qp);
1226 1231
1227 1232 ASSERT(qp->q_flag & QREADR);
1228 1233 entersq(qp->q_syncq, SQ_OPENCLOSE);
1229 1234
1230 1235 dummydev = *devp;
1231 1236 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1232 1237 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1233 1238 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1234 1239 mutex_enter(&STREAM(qp)->sd_lock);
1235 1240 qp->q_stream->sd_flag |= STREOPENFAIL;
1236 1241 mutex_exit(&STREAM(qp)->sd_lock);
1237 1242 return (error);
1238 1243 }
1239 1244 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1240 1245
1241 1246 /*
1242 1247 * successful open should have done qprocson()
1243 1248 */
1244 1249 ASSERT(qprocsareon(_RD(qp)));
1245 1250 return (0);
1246 1251 }
1247 1252
1248 1253 /*
1249 1254 * Detach a stream module or device.
1250 1255 * If clmode == 1 then the module or driver was opened and its
1251 1256 * close routine must be called. If clmode == 0, the module
1252 1257 * or driver was never opened or the open failed, and so its close
1253 1258 * should not be called.
1254 1259 */
1255 1260 void
1256 1261 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1257 1262 {
1258 1263 queue_t *wqp = _WR(qp);
1259 1264 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1260 1265
1261 1266 if (STREAM_NEEDSERVICE(STREAM(qp)))
1262 1267 stream_runservice(STREAM(qp));
1263 1268
1264 1269 if (clmode) {
1265 1270 /*
1266 1271 * Make sure that all the messages on the write side syncq are
1267 1272 * processed and nothing is left. Since we are closing, no new
1268 1273 * messages may appear there.
1269 1274 */
1270 1275 wait_q_syncq(wqp);
1271 1276
1272 1277 entersq(qp->q_syncq, SQ_OPENCLOSE);
1273 1278 if (is_remove) {
1274 1279 mutex_enter(QLOCK(qp));
1275 1280 qp->q_flag |= _QREMOVING;
1276 1281 mutex_exit(QLOCK(qp));
1277 1282 }
1278 1283 (*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1279 1284 /*
1280 1285 * Check that qprocsoff() was actually called.
1281 1286 */
1282 1287 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1283 1288
1284 1289 leavesq(qp->q_syncq, SQ_OPENCLOSE);
1285 1290 } else {
1286 1291 disable_svc(qp);
1287 1292 }
1288 1293
1289 1294 /*
1290 1295 * Allow any threads blocked in entersq to proceed and discover
1291 1296 * the QWCLOSE is set.
1292 1297 * Note: This assumes that all users of entersq check QWCLOSE.
1293 1298 * Currently runservice is the only entersq that can happen
1294 1299 * after removeq has finished.
1295 1300 * Removeq will have discarded all messages destined to the closing
1296 1301 * pair of queues from the syncq.
1297 1302 * NOTE: Calling a function inside an assert is unconventional.
1298 1303 * However, it does not cause any problem since flush_syncq() does
1299 1304 * not change any state except when it returns non-zero i.e.
1300 1305 * when the assert will trigger.
1301 1306 */
1302 1307 ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1303 1308 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1304 1309 ASSERT((qp->q_flag & QPERMOD) ||
1305 1310 ((qp->q_syncq->sq_head == NULL) &&
1306 1311 (wqp->q_syncq->sq_head == NULL)));
1307 1312
1308 1313 /* release any fmodsw_impl_t structure held on behalf of the queue */
1309 1314 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1310 1315 if (qp->q_fp != NULL)
1311 1316 fmodsw_rele(qp->q_fp);
1312 1317
1313 1318 /* freeq removes us from the outer perimeter if any */
1314 1319 freeq(qp);
1315 1320 }
1316 1321
1317 1322 /* Prevent service procedures from being called */
1318 1323 void
1319 1324 disable_svc(queue_t *qp)
1320 1325 {
1321 1326 queue_t *wqp = _WR(qp);
1322 1327
1323 1328 ASSERT(qp->q_flag & QREADR);
1324 1329 mutex_enter(QLOCK(qp));
1325 1330 qp->q_flag |= QWCLOSE;
1326 1331 mutex_exit(QLOCK(qp));
1327 1332 mutex_enter(QLOCK(wqp));
1328 1333 wqp->q_flag |= QWCLOSE;
1329 1334 mutex_exit(QLOCK(wqp));
1330 1335 }
1331 1336
1332 1337 /* Allow service procedures to be called again */
1333 1338 void
1334 1339 enable_svc(queue_t *qp)
1335 1340 {
1336 1341 queue_t *wqp = _WR(qp);
1337 1342
1338 1343 ASSERT(qp->q_flag & QREADR);
1339 1344 mutex_enter(QLOCK(qp));
1340 1345 qp->q_flag &= ~QWCLOSE;
1341 1346 mutex_exit(QLOCK(qp));
1342 1347 mutex_enter(QLOCK(wqp));
1343 1348 wqp->q_flag &= ~QWCLOSE;
1344 1349 mutex_exit(QLOCK(wqp));
1345 1350 }
1346 1351
1347 1352 /*
1348 1353 * Remove queue from qhead/qtail if it is enabled.
1349 1354 * Only reset QENAB if the queue was removed from the runlist.
1350 1355 * A queue goes through 3 stages:
1351 1356 * It is on the service list and QENAB is set.
1352 1357 * It is removed from the service list but QENAB is still set.
1353 1358 * QENAB gets changed to QINSERVICE.
1354 1359 * QINSERVICE is reset (when the service procedure is done)
1355 1360 * Thus we can not reset QENAB unless we actually removed it from the service
1356 1361 * queue.
1357 1362 */
1358 1363 void
1359 1364 remove_runlist(queue_t *qp)
1360 1365 {
1361 1366 if (qp->q_flag & QENAB && qhead != NULL) {
1362 1367 queue_t *q_chase;
1363 1368 queue_t *q_curr;
1364 1369 int removed;
1365 1370
1366 1371 mutex_enter(&service_queue);
1367 1372 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1368 1373 mutex_exit(&service_queue);
1369 1374 if (removed) {
1370 1375 STRSTAT(qremoved);
1371 1376 qp->q_flag &= ~QENAB;
1372 1377 }
1373 1378 }
1374 1379 }
1375 1380
1376 1381
1377 1382 /*
1378 1383 * Wait for any pending service processing to complete.
1379 1384 * The removal of queues from the runlist is not atomic with the
1380 1385 * clearing of the QENABLED flag and setting the INSERVICE flag.
1381 1386 * consequently it is possible for remove_runlist in strclose
1382 1387 * to not find the queue on the runlist but for it to be QENABLED
1383 1388 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1384 1389 * as well as INSERVICE.
1385 1390 */
1386 1391 void
1387 1392 wait_svc(queue_t *qp)
1388 1393 {
1389 1394 queue_t *wqp = _WR(qp);
1390 1395
1391 1396 ASSERT(qp->q_flag & QREADR);
1392 1397
1393 1398 /*
1394 1399 * Try to remove queues from qhead/qtail list.
1395 1400 */
1396 1401 if (qhead != NULL) {
1397 1402 remove_runlist(qp);
1398 1403 remove_runlist(wqp);
1399 1404 }
1400 1405 /*
1401 1406 * Wait till the syncqs associated with the queue disappear from the
1402 1407 * background processing list.
1403 1408 * This only needs to be done for non-PERMOD perimeters since
1404 1409 * for PERMOD perimeters the syncq may be shared and will only be freed
1405 1410 * when the last module/driver is unloaded.
1406 1411 * If for PERMOD perimeters queue was on the syncq list, removeq()
1407 1412 * should call propagate_syncq() or drain_syncq() for it. Both of these
1408 1413 * functions remove the queue from its syncq list, so sqthread will not
1409 1414 * try to access the queue.
1410 1415 */
1411 1416 if (!(qp->q_flag & QPERMOD)) {
1412 1417 syncq_t *rsq = qp->q_syncq;
1413 1418 syncq_t *wsq = wqp->q_syncq;
1414 1419
1415 1420 /*
1416 1421 * Disable rsq and wsq and wait for any background processing of
1417 1422 * syncq to complete.
1418 1423 */
1419 1424 wait_sq_svc(rsq);
1420 1425 if (wsq != rsq)
1421 1426 wait_sq_svc(wsq);
1422 1427 }
1423 1428
1424 1429 mutex_enter(QLOCK(qp));
1425 1430 while (qp->q_flag & (QINSERVICE|QENAB))
1426 1431 cv_wait(&qp->q_wait, QLOCK(qp));
1427 1432 mutex_exit(QLOCK(qp));
1428 1433 mutex_enter(QLOCK(wqp));
1429 1434 while (wqp->q_flag & (QINSERVICE|QENAB))
1430 1435 cv_wait(&wqp->q_wait, QLOCK(wqp));
1431 1436 mutex_exit(QLOCK(wqp));
1432 1437 }
1433 1438
1434 1439 /*
1435 1440 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1436 1441 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1437 1442 * also be set, and is passed through to allocb_cred_wait().
1438 1443 *
1439 1444 * Returns errno on failure, zero on success.
1440 1445 */
1441 1446 int
1442 1447 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1443 1448 {
1444 1449 mblk_t *tmp;
1445 1450 ssize_t count;
1446 1451 int error = 0;
1447 1452
1448 1453 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1449 1454 (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1450 1455
1451 1456 if (bp->b_datap->db_type == M_IOCTL) {
1452 1457 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1453 1458 } else {
1454 1459 ASSERT(bp->b_datap->db_type == M_COPYIN);
1455 1460 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1456 1461 }
1457 1462 /*
1458 1463 * strdoioctl validates ioc_count, so if this assert fails it
1459 1464 * cannot be due to user error.
1460 1465 */
1461 1466 ASSERT(count >= 0);
1462 1467
1463 1468 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1464 1469 curproc->p_pid)) == NULL) {
1465 1470 return (error);
1466 1471 }
1467 1472 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1468 1473 if (error != 0) {
1469 1474 freeb(tmp);
1470 1475 return (error);
1471 1476 }
1472 1477 DB_CPID(tmp) = curproc->p_pid;
1473 1478 tmp->b_wptr += count;
1474 1479 bp->b_cont = tmp;
1475 1480
1476 1481 return (0);
1477 1482 }
1478 1483
1479 1484 /*
1480 1485 * Copy ioctl data to user-land. Return non-zero errno on failure,
1481 1486 * 0 for success.
1482 1487 */
1483 1488 int
1484 1489 getiocd(mblk_t *bp, char *arg, int copymode)
1485 1490 {
1486 1491 ssize_t count;
1487 1492 size_t n;
1488 1493 int error;
1489 1494
1490 1495 if (bp->b_datap->db_type == M_IOCACK)
1491 1496 count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1492 1497 else {
1493 1498 ASSERT(bp->b_datap->db_type == M_COPYOUT);
1494 1499 count = ((struct copyreq *)bp->b_rptr)->cq_size;
1495 1500 }
1496 1501 ASSERT(count >= 0);
1497 1502
1498 1503 for (bp = bp->b_cont; bp && count;
1499 1504 count -= n, bp = bp->b_cont, arg += n) {
1500 1505 n = MIN(count, bp->b_wptr - bp->b_rptr);
1501 1506 error = strcopyout(bp->b_rptr, arg, n, copymode);
1502 1507 if (error)
1503 1508 return (error);
1504 1509 }
1505 1510 ASSERT(count == 0);
1506 1511 return (0);
1507 1512 }
1508 1513
1509 1514 /*
1510 1515 * Allocate a linkinfo entry given the write queue of the
1511 1516 * bottom module of the top stream and the write queue of the
1512 1517 * stream head of the bottom stream.
1513 1518 */
1514 1519 linkinfo_t *
1515 1520 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1516 1521 {
1517 1522 linkinfo_t *linkp;
1518 1523
1519 1524 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1520 1525
1521 1526 linkp->li_lblk.l_qtop = qup;
1522 1527 linkp->li_lblk.l_qbot = qdown;
1523 1528 linkp->li_fpdown = fpdown;
1524 1529
1525 1530 mutex_enter(&strresources);
1526 1531 linkp->li_next = linkinfo_list;
1527 1532 linkp->li_prev = NULL;
1528 1533 if (linkp->li_next)
1529 1534 linkp->li_next->li_prev = linkp;
1530 1535 linkinfo_list = linkp;
1531 1536 linkp->li_lblk.l_index = ++lnk_id;
1532 1537 ASSERT(lnk_id != 0); /* this should never wrap in practice */
1533 1538 mutex_exit(&strresources);
1534 1539
1535 1540 return (linkp);
1536 1541 }
1537 1542
1538 1543 /*
1539 1544 * Free a linkinfo entry.
1540 1545 */
1541 1546 void
1542 1547 lbfree(linkinfo_t *linkp)
1543 1548 {
1544 1549 mutex_enter(&strresources);
1545 1550 if (linkp->li_next)
1546 1551 linkp->li_next->li_prev = linkp->li_prev;
1547 1552 if (linkp->li_prev)
1548 1553 linkp->li_prev->li_next = linkp->li_next;
1549 1554 else
1550 1555 linkinfo_list = linkp->li_next;
1551 1556 mutex_exit(&strresources);
1552 1557
1553 1558 kmem_cache_free(linkinfo_cache, linkp);
1554 1559 }
1555 1560
1556 1561 /*
1557 1562 * Check for a potential linking cycle.
1558 1563 * Return 1 if a link will result in a cycle,
1559 1564 * and 0 otherwise.
1560 1565 */
1561 1566 int
1562 1567 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1563 1568 {
1564 1569 struct mux_node *np;
1565 1570 struct mux_edge *ep;
1566 1571 int i;
1567 1572 major_t lomaj;
1568 1573 major_t upmaj;
1569 1574 /*
1570 1575 * if the lower stream is a pipe/FIFO, return, since link
1571 1576 * cycles can not happen on pipes/FIFOs
1572 1577 */
1573 1578 if (lostp->sd_vnode->v_type == VFIFO)
1574 1579 return (0);
1575 1580
1576 1581 for (i = 0; i < ss->ss_devcnt; i++) {
1577 1582 np = &ss->ss_mux_nodes[i];
1578 1583 MUX_CLEAR(np);
1579 1584 }
1580 1585 lomaj = getmajor(lostp->sd_vnode->v_rdev);
1581 1586 upmaj = getmajor(upstp->sd_vnode->v_rdev);
1582 1587 np = &ss->ss_mux_nodes[lomaj];
1583 1588 for (;;) {
1584 1589 if (!MUX_DIDVISIT(np)) {
1585 1590 if (np->mn_imaj == upmaj)
1586 1591 return (1);
1587 1592 if (np->mn_outp == NULL) {
1588 1593 MUX_VISIT(np);
1589 1594 if (np->mn_originp == NULL)
1590 1595 return (0);
1591 1596 np = np->mn_originp;
1592 1597 continue;
1593 1598 }
1594 1599 MUX_VISIT(np);
1595 1600 np->mn_startp = np->mn_outp;
1596 1601 } else {
1597 1602 if (np->mn_startp == NULL) {
1598 1603 if (np->mn_originp == NULL)
1599 1604 return (0);
1600 1605 else {
1601 1606 np = np->mn_originp;
1602 1607 continue;
1603 1608 }
1604 1609 }
1605 1610 /*
1606 1611 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1607 1612 * ignore the edge and move on. ep->me_nodep gets
1608 1613 * set to NULL in mux_addedge() if it is a FIFO.
1609 1614 *
1610 1615 */
1611 1616 ep = np->mn_startp;
1612 1617 np->mn_startp = ep->me_nextp;
1613 1618 if (ep->me_nodep == NULL)
1614 1619 continue;
1615 1620 ep->me_nodep->mn_originp = np;
1616 1621 np = ep->me_nodep;
1617 1622 }
1618 1623 }
1619 1624 }
1620 1625
1621 1626 /*
1622 1627 * Find linkinfo entry corresponding to the parameters.
1623 1628 */
1624 1629 linkinfo_t *
1625 1630 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1626 1631 {
1627 1632 linkinfo_t *linkp;
1628 1633 struct mux_edge *mep;
1629 1634 struct mux_node *mnp;
1630 1635 queue_t *qup;
1631 1636
1632 1637 mutex_enter(&strresources);
1633 1638 if ((type & LINKTYPEMASK) == LINKNORMAL) {
1634 1639 qup = getendq(stp->sd_wrq);
1635 1640 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1636 1641 if ((qup == linkp->li_lblk.l_qtop) &&
1637 1642 (!index || (index == linkp->li_lblk.l_index))) {
1638 1643 mutex_exit(&strresources);
1639 1644 return (linkp);
1640 1645 }
1641 1646 }
1642 1647 } else {
1643 1648 ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1644 1649 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1645 1650 mep = mnp->mn_outp;
1646 1651 while (mep) {
1647 1652 if ((index == 0) || (index == mep->me_muxid))
1648 1653 break;
1649 1654 mep = mep->me_nextp;
1650 1655 }
1651 1656 if (!mep) {
1652 1657 mutex_exit(&strresources);
1653 1658 return (NULL);
1654 1659 }
1655 1660 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1656 1661 if ((!linkp->li_lblk.l_qtop) &&
1657 1662 (mep->me_muxid == linkp->li_lblk.l_index)) {
1658 1663 mutex_exit(&strresources);
1659 1664 return (linkp);
1660 1665 }
1661 1666 }
1662 1667 }
1663 1668 mutex_exit(&strresources);
1664 1669 return (NULL);
1665 1670 }
1666 1671
1667 1672 /*
1668 1673 * Given a queue ptr, follow the chain of q_next pointers until you reach the
1669 1674 * last queue on the chain and return it.
1670 1675 */
1671 1676 queue_t *
1672 1677 getendq(queue_t *q)
1673 1678 {
1674 1679 ASSERT(q != NULL);
1675 1680 while (_SAMESTR(q))
1676 1681 q = q->q_next;
1677 1682 return (q);
1678 1683 }
1679 1684
1680 1685 /*
1681 1686 * Wait for the syncq count to drop to zero.
1682 1687 * sq could be either outer or inner.
1683 1688 */
1684 1689
1685 1690 static void
1686 1691 wait_syncq(syncq_t *sq)
1687 1692 {
1688 1693 uint16_t count;
1689 1694
1690 1695 mutex_enter(SQLOCK(sq));
1691 1696 count = sq->sq_count;
1692 1697 SQ_PUTLOCKS_ENTER(sq);
1693 1698 SUM_SQ_PUTCOUNTS(sq, count);
1694 1699 while (count != 0) {
1695 1700 sq->sq_flags |= SQ_WANTWAKEUP;
1696 1701 SQ_PUTLOCKS_EXIT(sq);
1697 1702 cv_wait(&sq->sq_wait, SQLOCK(sq));
1698 1703 count = sq->sq_count;
1699 1704 SQ_PUTLOCKS_ENTER(sq);
1700 1705 SUM_SQ_PUTCOUNTS(sq, count);
1701 1706 }
1702 1707 SQ_PUTLOCKS_EXIT(sq);
1703 1708 mutex_exit(SQLOCK(sq));
1704 1709 }
1705 1710
1706 1711 /*
1707 1712 * Wait while there are any messages for the queue in its syncq.
1708 1713 */
1709 1714 static void
1710 1715 wait_q_syncq(queue_t *q)
1711 1716 {
1712 1717 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1713 1718 syncq_t *sq = q->q_syncq;
1714 1719
1715 1720 mutex_enter(SQLOCK(sq));
1716 1721 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1717 1722 sq->sq_flags |= SQ_WANTWAKEUP;
1718 1723 cv_wait(&sq->sq_wait, SQLOCK(sq));
1719 1724 }
1720 1725 mutex_exit(SQLOCK(sq));
1721 1726 }
1722 1727 }
1723 1728
1724 1729
1725 1730 int
1726 1731 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1727 1732 int lhlink)
1728 1733 {
1729 1734 struct stdata *stp;
1730 1735 struct strioctl strioc;
1731 1736 struct linkinfo *linkp;
1732 1737 struct stdata *stpdown;
1733 1738 struct streamtab *str;
1734 1739 queue_t *passq;
1735 1740 syncq_t *passyncq;
1736 1741 queue_t *rq;
1737 1742 cdevsw_impl_t *dp;
1738 1743 uint32_t qflag;
1739 1744 uint32_t sqtype;
1740 1745 perdm_t *dmp;
1741 1746 int error = 0;
1742 1747 netstack_t *ns;
1743 1748 str_stack_t *ss;
1744 1749
1745 1750 stp = vp->v_stream;
1746 1751 TRACE_1(TR_FAC_STREAMS_FR,
1747 1752 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1748 1753 /*
1749 1754 * Test for invalid upper stream
1750 1755 */
1751 1756 if (stp->sd_flag & STRHUP) {
1752 1757 return (ENXIO);
1753 1758 }
1754 1759 if (vp->v_type == VFIFO) {
1755 1760 return (EINVAL);
1756 1761 }
1757 1762 if (stp->sd_strtab == NULL) {
1758 1763 return (EINVAL);
1759 1764 }
1760 1765 if (!stp->sd_strtab->st_muxwinit) {
1761 1766 return (EINVAL);
1762 1767 }
1763 1768 if (fpdown == NULL) {
1764 1769 return (EBADF);
1765 1770 }
1766 1771 ns = netstack_find_by_cred(crp);
1767 1772 ASSERT(ns != NULL);
1768 1773 ss = ns->netstack_str;
1769 1774 ASSERT(ss != NULL);
1770 1775
1771 1776 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1772 1777 netstack_rele(ss->ss_netstack);
1773 1778 return (EINVAL);
1774 1779 }
1775 1780 mutex_enter(&muxifier);
1776 1781 if (stp->sd_flag & STPLEX) {
1777 1782 mutex_exit(&muxifier);
1778 1783 netstack_rele(ss->ss_netstack);
1779 1784 return (ENXIO);
1780 1785 }
1781 1786
1782 1787 /*
1783 1788 * Test for invalid lower stream.
1784 1789 * The check for the v_type != VFIFO and having a major
1785 1790 * number not >= devcnt is done to avoid problems with
1786 1791 * adding mux_node entry past the end of mux_nodes[].
1787 1792 * For FIFO's we don't add an entry so this isn't a
1788 1793 * problem.
1789 1794 */
1790 1795 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1791 1796 (stpdown == stp) || (stpdown->sd_flag &
1792 1797 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1793 1798 ((stpdown->sd_vnode->v_type != VFIFO) &&
1794 1799 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1795 1800 linkcycle(stp, stpdown, ss)) {
1796 1801 mutex_exit(&muxifier);
1797 1802 netstack_rele(ss->ss_netstack);
1798 1803 return (EINVAL);
1799 1804 }
1800 1805 TRACE_1(TR_FAC_STREAMS_FR,
1801 1806 TR_STPDOWN, "stpdown:%p", stpdown);
1802 1807 rq = getendq(stp->sd_wrq);
1803 1808 if (cmd == I_PLINK)
1804 1809 rq = NULL;
1805 1810
1806 1811 linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1807 1812
1808 1813 strioc.ic_cmd = cmd;
1809 1814 strioc.ic_timout = INFTIM;
1810 1815 strioc.ic_len = sizeof (struct linkblk);
1811 1816 strioc.ic_dp = (char *)&linkp->li_lblk;
1812 1817
1813 1818 /*
1814 1819 * STRPLUMB protects plumbing changes and should be set before
1815 1820 * link_addpassthru()/link_rempassthru() are called, so it is set here
1816 1821 * and cleared in the end of mlink when passthru queue is removed.
1817 1822 * Setting of STRPLUMB prevents reopens of the stream while passthru
1818 1823 * queue is in-place (it is not a proper module and doesn't have open
1819 1824 * entry point).
1820 1825 *
1821 1826 * STPLEX prevents any threads from entering the stream from above. It
1822 1827 * can't be set before the call to link_addpassthru() because putnext
1823 1828 * from below may cause stream head I/O routines to be called and these
1824 1829 * routines assert that STPLEX is not set. After link_addpassthru()
1825 1830 * nothing may come from below since the pass queue syncq is blocked.
1826 1831 * Note also that STPLEX should be cleared before the call to
1827 1832 * link_rempassthru() since when messages start flowing to the stream
1828 1833 * head (e.g. because of message propagation from the pass queue) stream
1829 1834 * head I/O routines may be called with STPLEX flag set.
1830 1835 *
1831 1836 * When STPLEX is set, nothing may come into the stream from above and
1832 1837 * it is safe to do a setq which will change stream head. So, the
1833 1838 * correct sequence of actions is:
1834 1839 *
1835 1840 * 1) Set STRPLUMB
1836 1841 * 2) Call link_addpassthru()
1837 1842 * 3) Set STPLEX
1838 1843 * 4) Call setq and update the stream state
1839 1844 * 5) Clear STPLEX
1840 1845 * 6) Call link_rempassthru()
1841 1846 * 7) Clear STRPLUMB
1842 1847 *
1843 1848 * The same sequence applies to munlink() code.
1844 1849 */
1845 1850 mutex_enter(&stpdown->sd_lock);
1846 1851 stpdown->sd_flag |= STRPLUMB;
1847 1852 mutex_exit(&stpdown->sd_lock);
1848 1853 /*
1849 1854 * Add passthru queue below lower mux. This will block
1850 1855 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1851 1856 */
1852 1857 passq = link_addpassthru(stpdown);
1853 1858
1854 1859 mutex_enter(&stpdown->sd_lock);
1855 1860 stpdown->sd_flag |= STPLEX;
1856 1861 mutex_exit(&stpdown->sd_lock);
1857 1862
1858 1863 rq = _RD(stpdown->sd_wrq);
1859 1864 /*
1860 1865 * There may be messages in the streamhead's syncq due to messages
1861 1866 * that arrived before link_addpassthru() was done. To avoid
1862 1867 * background processing of the syncq happening simultaneous with
1863 1868 * setq processing, we disable the streamhead syncq and wait until
1864 1869 * existing background thread finishes working on it.
1865 1870 */
1866 1871 wait_sq_svc(rq->q_syncq);
1867 1872 passyncq = passq->q_syncq;
1868 1873 if (!(passyncq->sq_flags & SQ_BLOCKED))
1869 1874 blocksq(passyncq, SQ_BLOCKED, 0);
1870 1875
1871 1876 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1872 1877 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1873 1878 rq->q_ptr = _WR(rq)->q_ptr = NULL;
1874 1879
1875 1880 /* setq might sleep in allocator - avoid holding locks. */
1876 1881 /* Note: we are holding muxifier here. */
1877 1882
1878 1883 str = stp->sd_strtab;
1879 1884 dp = &devimpl[getmajor(vp->v_rdev)];
1880 1885 ASSERT(dp->d_str == str);
1881 1886
1882 1887 qflag = dp->d_qflag;
1883 1888 sqtype = dp->d_sqtype;
1884 1889
1885 1890 /* create perdm_t if needed */
1886 1891 if (NEED_DM(dp->d_dmp, qflag))
1887 1892 dp->d_dmp = hold_dm(str, qflag, sqtype);
1888 1893
1889 1894 dmp = dp->d_dmp;
1890 1895
1891 1896 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1892 1897 B_TRUE);
1893 1898
1894 1899 /*
1895 1900 * XXX Remove any "odd" messages from the queue.
1896 1901 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1897 1902 */
1898 1903 error = strdoioctl(stp, &strioc, FNATIVE,
1899 1904 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1900 1905 if (error != 0) {
1901 1906 lbfree(linkp);
1902 1907
1903 1908 if (!(passyncq->sq_flags & SQ_BLOCKED))
1904 1909 blocksq(passyncq, SQ_BLOCKED, 0);
1905 1910 /*
1906 1911 * Restore the stream head queue and then remove
1907 1912 * the passq. Turn off STPLEX before we turn on
1908 1913 * the stream by removing the passq.
1909 1914 */
1910 1915 rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1911 1916 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1912 1917 B_TRUE);
1913 1918
1914 1919 mutex_enter(&stpdown->sd_lock);
1915 1920 stpdown->sd_flag &= ~STPLEX;
1916 1921 mutex_exit(&stpdown->sd_lock);
1917 1922
1918 1923 link_rempassthru(passq);
1919 1924
1920 1925 mutex_enter(&stpdown->sd_lock);
1921 1926 stpdown->sd_flag &= ~STRPLUMB;
1922 1927 /* Wakeup anyone waiting for STRPLUMB to clear. */
1923 1928 cv_broadcast(&stpdown->sd_monitor);
1924 1929 mutex_exit(&stpdown->sd_lock);
1925 1930
1926 1931 mutex_exit(&muxifier);
1927 1932 netstack_rele(ss->ss_netstack);
1928 1933 return (error);
1929 1934 }
1930 1935 mutex_enter(&fpdown->f_tlock);
1931 1936 fpdown->f_count++;
1932 1937 mutex_exit(&fpdown->f_tlock);
1933 1938
1934 1939 /*
1935 1940 * if we've made it here the linkage is all set up so we should also
1936 1941 * set up the layered driver linkages
1937 1942 */
1938 1943
1939 1944 ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1940 1945 if (cmd == I_LINK) {
1941 1946 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1942 1947 } else {
1943 1948 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1944 1949 }
1945 1950
1946 1951 link_rempassthru(passq);
1947 1952
1948 1953 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1949 1954
1950 1955 /*
1951 1956 * Mark the upper stream as having dependent links
1952 1957 * so that strclose can clean it up.
1953 1958 */
1954 1959 if (cmd == I_LINK) {
1955 1960 mutex_enter(&stp->sd_lock);
1956 1961 stp->sd_flag |= STRHASLINKS;
1957 1962 mutex_exit(&stp->sd_lock);
1958 1963 }
1959 1964 /*
1960 1965 * Wake up any other processes that may have been
1961 1966 * waiting on the lower stream. These will all
1962 1967 * error out.
1963 1968 */
1964 1969 mutex_enter(&stpdown->sd_lock);
1965 1970 /* The passthru module is removed so we may release STRPLUMB */
1966 1971 stpdown->sd_flag &= ~STRPLUMB;
1967 1972 cv_broadcast(&rq->q_wait);
1968 1973 cv_broadcast(&_WR(rq)->q_wait);
1969 1974 cv_broadcast(&stpdown->sd_monitor);
1970 1975 mutex_exit(&stpdown->sd_lock);
1971 1976 mutex_exit(&muxifier);
1972 1977 *rvalp = linkp->li_lblk.l_index;
1973 1978 netstack_rele(ss->ss_netstack);
1974 1979 return (0);
1975 1980 }
1976 1981
1977 1982 int
1978 1983 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1979 1984 {
1980 1985 int ret;
1981 1986 struct file *fpdown;
1982 1987
1983 1988 fpdown = getf(arg);
1984 1989 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1985 1990 if (fpdown != NULL)
1986 1991 releasef(arg);
1987 1992 return (ret);
1988 1993 }
1989 1994
1990 1995 /*
1991 1996 * Unlink a multiplexor link. Stp is the controlling stream for the
1992 1997 * link, and linkp points to the link's entry in the linkinfo list.
1993 1998 * The muxifier lock must be held on entry and is dropped on exit.
1994 1999 *
1995 2000 * NOTE : Currently it is assumed that mux would process all the messages
1996 2001 * sitting on it's queue before ACKing the UNLINK. It is the responsibility
1997 2002 * of the mux to handle all the messages that arrive before UNLINK.
1998 2003 * If the mux has to send down messages on its lower stream before
1999 2004 * ACKing I_UNLINK, then it *should* know to handle messages even
2000 2005 * after the UNLINK is acked (actually it should be able to handle till we
2001 2006 * re-block the read side of the pass queue here). If the mux does not
2002 2007 * open up the lower stream, any messages that arrive during UNLINK
2003 2008 * will be put in the stream head. In the case of lower stream opening
2004 2009 * up, some messages might land in the stream head depending on when
2005 2010 * the message arrived and when the read side of the pass queue was
2006 2011 * re-blocked.
2007 2012 */
2008 2013 int
2009 2014 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2010 2015 str_stack_t *ss)
2011 2016 {
2012 2017 struct strioctl strioc;
2013 2018 struct stdata *stpdown;
2014 2019 queue_t *rq, *wrq;
2015 2020 queue_t *passq;
2016 2021 syncq_t *passyncq;
2017 2022 int error = 0;
2018 2023 file_t *fpdown;
2019 2024
2020 2025 ASSERT(MUTEX_HELD(&muxifier));
2021 2026
2022 2027 stpdown = linkp->li_fpdown->f_vnode->v_stream;
2023 2028
2024 2029 /*
2025 2030 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2026 2031 */
2027 2032 mutex_enter(&stpdown->sd_lock);
2028 2033 stpdown->sd_flag |= STRPLUMB;
2029 2034 mutex_exit(&stpdown->sd_lock);
2030 2035
2031 2036 /*
2032 2037 * Add passthru queue below lower mux. This will block
2033 2038 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2034 2039 */
2035 2040 passq = link_addpassthru(stpdown);
2036 2041
2037 2042 if ((flag & LINKTYPEMASK) == LINKNORMAL)
2038 2043 strioc.ic_cmd = I_UNLINK;
2039 2044 else
2040 2045 strioc.ic_cmd = I_PUNLINK;
2041 2046 strioc.ic_timout = INFTIM;
2042 2047 strioc.ic_len = sizeof (struct linkblk);
2043 2048 strioc.ic_dp = (char *)&linkp->li_lblk;
2044 2049
2045 2050 error = strdoioctl(stp, &strioc, FNATIVE,
2046 2051 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2047 2052
2048 2053 /*
2049 2054 * If there was an error and this is not called via strclose,
2050 2055 * return to the user. Otherwise, pretend there was no error
2051 2056 * and close the link.
2052 2057 */
2053 2058 if (error) {
2054 2059 if (flag & LINKCLOSE) {
2055 2060 cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2056 2061 "unlink ioctl, closing anyway (%d)\n", error);
2057 2062 } else {
2058 2063 link_rempassthru(passq);
2059 2064 mutex_enter(&stpdown->sd_lock);
2060 2065 stpdown->sd_flag &= ~STRPLUMB;
2061 2066 cv_broadcast(&stpdown->sd_monitor);
2062 2067 mutex_exit(&stpdown->sd_lock);
2063 2068 mutex_exit(&muxifier);
2064 2069 return (error);
2065 2070 }
2066 2071 }
2067 2072
2068 2073 mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2069 2074 fpdown = linkp->li_fpdown;
2070 2075 lbfree(linkp);
2071 2076
2072 2077 /*
2073 2078 * We go ahead and drop muxifier here--it's a nasty global lock that
2074 2079 * can slow others down. It's okay to since attempts to mlink() this
2075 2080 * stream will be stopped because STPLEX is still set in the stdata
2076 2081 * structure, and munlink() is stopped because mux_rmvedge() and
2077 2082 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2078 2083 * respectively. Note that we defer the closef() of fpdown until
2079 2084 * after we drop muxifier since strclose() can call munlinkall().
2080 2085 */
2081 2086 mutex_exit(&muxifier);
2082 2087
2083 2088 wrq = stpdown->sd_wrq;
2084 2089 rq = _RD(wrq);
2085 2090
2086 2091 /*
2087 2092 * Get rid of outstanding service procedure runs, before we make
2088 2093 * it a stream head, since a stream head doesn't have any service
2089 2094 * procedure.
2090 2095 */
2091 2096 disable_svc(rq);
2092 2097 wait_svc(rq);
2093 2098
2094 2099 /*
2095 2100 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2096 2101 * is queued up to be finished. mux should take care that nothing is
2097 2102 * send down to this queue. We should do it now as we're going to block
2098 2103 * passyncq if it was unblocked.
2099 2104 */
2100 2105 if (wrq->q_flag & QPERMOD) {
2101 2106 syncq_t *sq = wrq->q_syncq;
2102 2107
2103 2108 mutex_enter(SQLOCK(sq));
2104 2109 while (wrq->q_sqflags & Q_SQQUEUED) {
2105 2110 sq->sq_flags |= SQ_WANTWAKEUP;
2106 2111 cv_wait(&sq->sq_wait, SQLOCK(sq));
2107 2112 }
2108 2113 mutex_exit(SQLOCK(sq));
2109 2114 }
2110 2115 passyncq = passq->q_syncq;
2111 2116 if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2112 2117
2113 2118 syncq_t *sq, *outer;
2114 2119
2115 2120 /*
2116 2121 * Messages could be flowing from underneath. We will
2117 2122 * block the read side of the passq. This would be
2118 2123 * sufficient for QPAIR and QPERQ muxes to ensure
2119 2124 * that no data is flowing up into this queue
2120 2125 * and hence no thread active in this instance of
2121 2126 * lower mux. But for QPERMOD and QMTOUTPERIM there
2122 2127 * could be messages on the inner and outer/inner
2123 2128 * syncqs respectively. We will wait for them to drain.
2124 2129 * Because passq is blocked messages end up in the syncq
2125 2130 * And qfill_syncq could possibly end up setting QFULL
2126 2131 * which will access the rq->q_flag. Hence, we have to
2127 2132 * acquire the QLOCK in setq.
2128 2133 *
2129 2134 * XXX Messages can also flow from top into this
2130 2135 * queue though the unlink is over (Ex. some instance
2131 2136 * in putnext() called from top that has still not
2132 2137 * accessed this queue. And also putq(lowerq) ?).
2133 2138 * Solution : How about blocking the l_qtop queue ?
2134 2139 * Do we really care about such pure D_MP muxes ?
2135 2140 */
2136 2141
2137 2142 blocksq(passyncq, SQ_BLOCKED, 0);
2138 2143
2139 2144 sq = rq->q_syncq;
2140 2145 if ((outer = sq->sq_outer) != NULL) {
2141 2146
2142 2147 /*
2143 2148 * We have to just wait for the outer sq_count
2144 2149 * drop to zero. As this does not prevent new
2145 2150 * messages to enter the outer perimeter, this
2146 2151 * is subject to starvation.
2147 2152 *
2148 2153 * NOTE :Because of blocksq above, messages could
2149 2154 * be in the inner syncq only because of some
2150 2155 * thread holding the outer perimeter exclusively.
2151 2156 * Hence it would be sufficient to wait for the
2152 2157 * exclusive holder of the outer perimeter to drain
2153 2158 * the inner and outer syncqs. But we will not depend
2154 2159 * on this feature and hence check the inner syncqs
2155 2160 * separately.
2156 2161 */
2157 2162 wait_syncq(outer);
2158 2163 }
2159 2164
2160 2165
2161 2166 /*
2162 2167 * There could be messages destined for
2163 2168 * this queue. Let the exclusive holder
2164 2169 * drain it.
2165 2170 */
2166 2171
2167 2172 wait_syncq(sq);
2168 2173 ASSERT((rq->q_flag & QPERMOD) ||
2169 2174 ((rq->q_syncq->sq_head == NULL) &&
2170 2175 (_WR(rq)->q_syncq->sq_head == NULL)));
2171 2176 }
2172 2177
2173 2178 /*
2174 2179 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2175 2180 * case as we don't disable its syncq or remove it off the syncq
2176 2181 * service list.
2177 2182 */
2178 2183 if (rq->q_flag & QPERMOD) {
2179 2184 syncq_t *sq = rq->q_syncq;
2180 2185
2181 2186 mutex_enter(SQLOCK(sq));
2182 2187 while (rq->q_sqflags & Q_SQQUEUED) {
2183 2188 sq->sq_flags |= SQ_WANTWAKEUP;
2184 2189 cv_wait(&sq->sq_wait, SQLOCK(sq));
2185 2190 }
2186 2191 mutex_exit(SQLOCK(sq));
2187 2192 }
2188 2193
2189 2194 /*
2190 2195 * flush_syncq changes states only when there are some messages to
2191 2196 * free, i.e. when it returns non-zero value to return.
2192 2197 */
2193 2198 ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2194 2199 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2195 2200
2196 2201 /*
2197 2202 * Nobody else should know about this queue now.
2198 2203 * If the mux did not process the messages before
2199 2204 * acking the I_UNLINK, free them now.
2200 2205 */
2201 2206
2202 2207 flushq(rq, FLUSHALL);
2203 2208 flushq(_WR(rq), FLUSHALL);
2204 2209
2205 2210 /*
2206 2211 * Convert the mux lower queue into a stream head queue.
2207 2212 * Turn off STPLEX before we turn on the stream by removing the passq.
2208 2213 */
2209 2214 rq->q_ptr = wrq->q_ptr = stpdown;
2210 2215 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2211 2216
2212 2217 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2213 2218 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2214 2219
2215 2220 enable_svc(rq);
2216 2221
2217 2222 /*
2218 2223 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2219 2224 * needs to be set to prevent reopen() of the stream - such reopen may
2220 2225 * try to call non-existent pass queue open routine and panic.
2221 2226 */
2222 2227 mutex_enter(&stpdown->sd_lock);
2223 2228 stpdown->sd_flag &= ~STPLEX;
2224 2229 mutex_exit(&stpdown->sd_lock);
2225 2230
2226 2231 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2227 2232 ((flag & LINKTYPEMASK) == LINKPERSIST));
2228 2233
2229 2234 /* clean up the layered driver linkages */
2230 2235 if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2231 2236 ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2232 2237 } else {
2233 2238 ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2234 2239 }
2235 2240
2236 2241 link_rempassthru(passq);
2237 2242
2238 2243 /*
2239 2244 * Now all plumbing changes are finished and STRPLUMB is no
2240 2245 * longer needed.
2241 2246 */
2242 2247 mutex_enter(&stpdown->sd_lock);
2243 2248 stpdown->sd_flag &= ~STRPLUMB;
2244 2249 cv_broadcast(&stpdown->sd_monitor);
2245 2250 mutex_exit(&stpdown->sd_lock);
2246 2251
2247 2252 (void) closef(fpdown);
2248 2253 return (0);
2249 2254 }
2250 2255
2251 2256 /*
2252 2257 * Unlink all multiplexor links for which stp is the controlling stream.
2253 2258 * Return 0, or a non-zero errno on failure.
2254 2259 */
2255 2260 int
2256 2261 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2257 2262 {
2258 2263 linkinfo_t *linkp;
2259 2264 int error = 0;
2260 2265
2261 2266 mutex_enter(&muxifier);
2262 2267 while (linkp = findlinks(stp, 0, flag, ss)) {
2263 2268 /*
2264 2269 * munlink() releases the muxifier lock.
2265 2270 */
2266 2271 if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2267 2272 return (error);
2268 2273 mutex_enter(&muxifier);
2269 2274 }
2270 2275 mutex_exit(&muxifier);
2271 2276 return (0);
2272 2277 }
2273 2278
2274 2279 /*
2275 2280 * A multiplexor link has been made. Add an
2276 2281 * edge to the directed graph.
2277 2282 */
2278 2283 void
2279 2284 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2280 2285 {
2281 2286 struct mux_node *np;
2282 2287 struct mux_edge *ep;
2283 2288 major_t upmaj;
2284 2289 major_t lomaj;
2285 2290
2286 2291 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2287 2292 lomaj = getmajor(lostp->sd_vnode->v_rdev);
2288 2293 np = &ss->ss_mux_nodes[upmaj];
2289 2294 if (np->mn_outp) {
2290 2295 ep = np->mn_outp;
2291 2296 while (ep->me_nextp)
2292 2297 ep = ep->me_nextp;
2293 2298 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2294 2299 ep = ep->me_nextp;
2295 2300 } else {
2296 2301 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2297 2302 ep = np->mn_outp;
2298 2303 }
2299 2304 ep->me_nextp = NULL;
2300 2305 ep->me_muxid = muxid;
2301 2306 /*
2302 2307 * Save the dev_t for the purposes of str_stack_shutdown.
2303 2308 * str_stack_shutdown assumes that the device allows reopen, since
2304 2309 * this dev_t is the one after any cloning by xx_open().
2305 2310 * Would prefer finding the dev_t from before any cloning,
2306 2311 * but specfs doesn't retain that.
2307 2312 */
2308 2313 ep->me_dev = upstp->sd_vnode->v_rdev;
2309 2314 if (lostp->sd_vnode->v_type == VFIFO)
2310 2315 ep->me_nodep = NULL;
2311 2316 else
2312 2317 ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2313 2318 }
2314 2319
2315 2320 /*
2316 2321 * A multiplexor link has been removed. Remove the
2317 2322 * edge in the directed graph.
2318 2323 */
2319 2324 void
2320 2325 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2321 2326 {
2322 2327 struct mux_node *np;
2323 2328 struct mux_edge *ep;
2324 2329 struct mux_edge *pep = NULL;
2325 2330 major_t upmaj;
2326 2331
2327 2332 upmaj = getmajor(upstp->sd_vnode->v_rdev);
2328 2333 np = &ss->ss_mux_nodes[upmaj];
2329 2334 ASSERT(np->mn_outp != NULL);
2330 2335 ep = np->mn_outp;
2331 2336 while (ep) {
2332 2337 if (ep->me_muxid == muxid) {
2333 2338 if (pep)
2334 2339 pep->me_nextp = ep->me_nextp;
2335 2340 else
2336 2341 np->mn_outp = ep->me_nextp;
2337 2342 kmem_free(ep, sizeof (struct mux_edge));
2338 2343 return;
2339 2344 }
2340 2345 pep = ep;
2341 2346 ep = ep->me_nextp;
↓ open down ↓ |
1379 lines elided |
↑ open up ↑ |
2342 2347 }
2343 2348 ASSERT(0); /* should not reach here */
2344 2349 }
2345 2350
2346 2351 /*
2347 2352 * Translate the device flags (from conf.h) to the corresponding
2348 2353 * qflag and sq_flag (type) values.
2349 2354 */
2350 2355 int
2351 2356 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2352 - uint32_t *sqtypep)
2357 + uint32_t *sqtypep)
2353 2358 {
2354 2359 uint32_t qflag = 0;
2355 2360 uint32_t sqtype = 0;
2356 2361
2357 2362 if (devflag & _D_OLD)
2358 2363 goto bad;
2359 2364
2360 2365 /* Inner perimeter presence and scope */
2361 2366 switch (devflag & D_MTINNER_MASK) {
2362 2367 case D_MP:
2363 2368 qflag |= QMTSAFE;
2364 2369 sqtype |= SQ_CI;
2365 2370 break;
2366 2371 case D_MTPERQ|D_MP:
2367 2372 qflag |= QPERQ;
2368 2373 break;
2369 2374 case D_MTQPAIR|D_MP:
2370 2375 qflag |= QPAIR;
2371 2376 break;
2372 2377 case D_MTPERMOD|D_MP:
2373 2378 qflag |= QPERMOD;
2374 2379 break;
2375 2380 default:
2376 2381 goto bad;
2377 2382 }
2378 2383
2379 2384 /* Outer perimeter */
2380 2385 if (devflag & D_MTOUTPERIM) {
2381 2386 switch (devflag & D_MTINNER_MASK) {
2382 2387 case D_MP:
2383 2388 case D_MTPERQ|D_MP:
2384 2389 case D_MTQPAIR|D_MP:
2385 2390 break;
2386 2391 default:
2387 2392 goto bad;
2388 2393 }
2389 2394 qflag |= QMTOUTPERIM;
2390 2395 }
2391 2396
2392 2397 /* Inner perimeter modifiers */
2393 2398 if (devflag & D_MTINNER_MOD) {
2394 2399 switch (devflag & D_MTINNER_MASK) {
2395 2400 case D_MP:
2396 2401 goto bad;
2397 2402 default:
2398 2403 break;
2399 2404 }
2400 2405 if (devflag & D_MTPUTSHARED)
2401 2406 sqtype |= SQ_CIPUT;
2402 2407 if (devflag & _D_MTOCSHARED) {
2403 2408 /*
2404 2409 * The code in putnext assumes that it has the
2405 2410 * highest concurrency by not checking sq_count.
2406 2411 * Thus _D_MTOCSHARED can only be supported when
2407 2412 * D_MTPUTSHARED is set.
2408 2413 */
2409 2414 if (!(devflag & D_MTPUTSHARED))
2410 2415 goto bad;
2411 2416 sqtype |= SQ_CIOC;
2412 2417 }
2413 2418 if (devflag & _D_MTCBSHARED) {
2414 2419 /*
2415 2420 * The code in putnext assumes that it has the
2416 2421 * highest concurrency by not checking sq_count.
2417 2422 * Thus _D_MTCBSHARED can only be supported when
2418 2423 * D_MTPUTSHARED is set.
2419 2424 */
2420 2425 if (!(devflag & D_MTPUTSHARED))
2421 2426 goto bad;
2422 2427 sqtype |= SQ_CICB;
2423 2428 }
2424 2429 if (devflag & _D_MTSVCSHARED) {
2425 2430 /*
2426 2431 * The code in putnext assumes that it has the
2427 2432 * highest concurrency by not checking sq_count.
2428 2433 * Thus _D_MTSVCSHARED can only be supported when
2429 2434 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2430 2435 * supported only for QPERMOD.
2431 2436 */
2432 2437 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2433 2438 goto bad;
2434 2439 sqtype |= SQ_CISVC;
2435 2440 }
2436 2441 }
2437 2442
2438 2443 /* Default outer perimeter concurrency */
2439 2444 sqtype |= SQ_CO;
2440 2445
2441 2446 /* Outer perimeter modifiers */
2442 2447 if (devflag & D_MTOCEXCL) {
2443 2448 if (!(devflag & D_MTOUTPERIM)) {
2444 2449 /* No outer perimeter */
2445 2450 goto bad;
2446 2451 }
2447 2452 sqtype &= ~SQ_COOC;
2448 2453 }
2449 2454
2450 2455 /* Synchronous Streams extended qinit structure */
2451 2456 if (devflag & D_SYNCSTR)
2452 2457 qflag |= QSYNCSTR;
2453 2458
2454 2459 /*
2455 2460 * Private flag used by a transport module to indicate
2456 2461 * to sockfs that it supports direct-access mode without
2457 2462 * having to go through STREAMS.
2458 2463 */
2459 2464 if (devflag & _D_DIRECT) {
2460 2465 /* Reject unless the module is fully-MT (no perimeter) */
2461 2466 if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2462 2467 goto bad;
2463 2468 qflag |= _QDIRECT;
2464 2469 }
2465 2470
2466 2471 *qflagp = qflag;
2467 2472 *sqtypep = sqtype;
2468 2473 return (0);
2469 2474
2470 2475 bad:
2471 2476 cmn_err(CE_WARN,
2472 2477 "stropen: bad MT flags (0x%x) in driver '%s'",
2473 2478 (int)(qflag & D_MTSAFETY_MASK),
2474 2479 stp->st_rdinit->qi_minfo->mi_idname);
2475 2480
2476 2481 return (EINVAL);
2477 2482 }
2478 2483
2479 2484 /*
2480 2485 * Set the interface values for a pair of queues (qinit structure,
2481 2486 * packet sizes, water marks).
2482 2487 * setq assumes that the caller does not have a claim (entersq or claimq)
2483 2488 * on the queue.
2484 2489 */
2485 2490 void
2486 2491 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2487 2492 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2488 2493 {
2489 2494 queue_t *wq;
2490 2495 syncq_t *sq, *outer;
2491 2496
2492 2497 ASSERT(rq->q_flag & QREADR);
2493 2498 ASSERT((qflag & QMT_TYPEMASK) != 0);
2494 2499 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2495 2500
2496 2501 wq = _WR(rq);
2497 2502 rq->q_qinfo = rinit;
2498 2503 rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2499 2504 rq->q_lowat = rinit->qi_minfo->mi_lowat;
2500 2505 rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2501 2506 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2502 2507 wq->q_qinfo = winit;
2503 2508 wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2504 2509 wq->q_lowat = winit->qi_minfo->mi_lowat;
2505 2510 wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2506 2511 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2507 2512
2508 2513 /* Remove old syncqs */
2509 2514 sq = rq->q_syncq;
2510 2515 outer = sq->sq_outer;
2511 2516 if (outer != NULL) {
2512 2517 ASSERT(wq->q_syncq->sq_outer == outer);
2513 2518 outer_remove(outer, rq->q_syncq);
2514 2519 if (wq->q_syncq != rq->q_syncq)
2515 2520 outer_remove(outer, wq->q_syncq);
2516 2521 }
2517 2522 ASSERT(sq->sq_outer == NULL);
2518 2523 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2519 2524
2520 2525 if (sq != SQ(rq)) {
2521 2526 if (!(rq->q_flag & QPERMOD))
2522 2527 free_syncq(sq);
2523 2528 if (wq->q_syncq == rq->q_syncq)
2524 2529 wq->q_syncq = NULL;
2525 2530 rq->q_syncq = NULL;
2526 2531 }
2527 2532 if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2528 2533 wq->q_syncq != SQ(rq)) {
2529 2534 free_syncq(wq->q_syncq);
2530 2535 wq->q_syncq = NULL;
2531 2536 }
2532 2537 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2533 2538 rq->q_syncq->sq_tail == NULL));
2534 2539 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2535 2540 wq->q_syncq->sq_tail == NULL));
2536 2541
2537 2542 if (!(rq->q_flag & QPERMOD) &&
2538 2543 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2539 2544 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2540 2545 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2541 2546 rq->q_syncq->sq_nciputctrl, 0);
2542 2547 ASSERT(ciputctrl_cache != NULL);
2543 2548 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2544 2549 rq->q_syncq->sq_ciputctrl = NULL;
2545 2550 rq->q_syncq->sq_nciputctrl = 0;
2546 2551 }
2547 2552
2548 2553 if (!(wq->q_flag & QPERMOD) &&
2549 2554 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2550 2555 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2551 2556 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2552 2557 wq->q_syncq->sq_nciputctrl, 0);
2553 2558 ASSERT(ciputctrl_cache != NULL);
2554 2559 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2555 2560 wq->q_syncq->sq_ciputctrl = NULL;
2556 2561 wq->q_syncq->sq_nciputctrl = 0;
2557 2562 }
2558 2563
2559 2564 sq = SQ(rq);
2560 2565 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2561 2566 ASSERT(sq->sq_outer == NULL);
2562 2567 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2563 2568
2564 2569 /*
2565 2570 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2566 2571 * bits in sq_flag based on the sqtype.
2567 2572 */
2568 2573 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2569 2574
2570 2575 rq->q_syncq = wq->q_syncq = sq;
2571 2576 sq->sq_type = sqtype;
2572 2577 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2573 2578
2574 2579 /*
2575 2580 * We are making sq_svcflags zero,
2576 2581 * resetting SQ_DISABLED in case it was set by
2577 2582 * wait_svc() in the munlink path.
2578 2583 *
2579 2584 */
2580 2585 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2581 2586 sq->sq_svcflags = 0;
2582 2587
2583 2588 /*
2584 2589 * We need to acquire the lock here for the mlink and munlink case,
2585 2590 * where canputnext, backenable, etc can access the q_flag.
2586 2591 */
2587 2592 if (lock_needed) {
2588 2593 mutex_enter(QLOCK(rq));
2589 2594 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2590 2595 mutex_exit(QLOCK(rq));
2591 2596 mutex_enter(QLOCK(wq));
2592 2597 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2593 2598 mutex_exit(QLOCK(wq));
2594 2599 } else {
2595 2600 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2596 2601 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2597 2602 }
2598 2603
2599 2604 if (qflag & QPERQ) {
2600 2605 /* Allocate a separate syncq for the write side */
2601 2606 sq = new_syncq();
2602 2607 sq->sq_type = rq->q_syncq->sq_type;
2603 2608 sq->sq_flags = rq->q_syncq->sq_flags;
2604 2609 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2605 2610 sq->sq_oprev == NULL);
2606 2611 wq->q_syncq = sq;
2607 2612 }
2608 2613 if (qflag & QPERMOD) {
2609 2614 sq = dmp->dm_sq;
2610 2615
2611 2616 /*
2612 2617 * Assert that we do have an inner perimeter syncq and that it
2613 2618 * does not have an outer perimeter associated with it.
2614 2619 */
2615 2620 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2616 2621 sq->sq_oprev == NULL);
2617 2622 rq->q_syncq = wq->q_syncq = sq;
2618 2623 }
2619 2624 if (qflag & QMTOUTPERIM) {
2620 2625 outer = dmp->dm_sq;
2621 2626
2622 2627 ASSERT(outer->sq_outer == NULL);
2623 2628 outer_insert(outer, rq->q_syncq);
2624 2629 if (wq->q_syncq != rq->q_syncq)
2625 2630 outer_insert(outer, wq->q_syncq);
2626 2631 }
2627 2632 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2628 2633 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2629 2634 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2630 2635 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2631 2636 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2632 2637
2633 2638 /*
2634 2639 * Initialize struio() types.
2635 2640 */
2636 2641 rq->q_struiot =
2637 2642 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2638 2643 wq->q_struiot =
2639 2644 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2640 2645 }
2641 2646
2642 2647 perdm_t *
2643 2648 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2644 2649 {
2645 2650 syncq_t *sq;
2646 2651 perdm_t **pp;
2647 2652 perdm_t *p;
2648 2653 perdm_t *dmp;
2649 2654
2650 2655 ASSERT(str != NULL);
2651 2656 ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2652 2657
2653 2658 rw_enter(&perdm_rwlock, RW_READER);
2654 2659 for (p = perdm_list; p != NULL; p = p->dm_next) {
2655 2660 if (p->dm_str == str) { /* found one */
2656 2661 atomic_inc_32(&(p->dm_ref));
2657 2662 rw_exit(&perdm_rwlock);
2658 2663 return (p);
2659 2664 }
2660 2665 }
2661 2666 rw_exit(&perdm_rwlock);
2662 2667
2663 2668 sq = new_syncq();
2664 2669 if (qflag & QPERMOD) {
2665 2670 sq->sq_type = sqtype | SQ_PERMOD;
2666 2671 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2667 2672 } else {
2668 2673 ASSERT(qflag & QMTOUTPERIM);
2669 2674 sq->sq_onext = sq->sq_oprev = sq;
2670 2675 }
2671 2676
2672 2677 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2673 2678 dmp->dm_sq = sq;
2674 2679 dmp->dm_str = str;
2675 2680 dmp->dm_ref = 1;
2676 2681 dmp->dm_next = NULL;
2677 2682
2678 2683 rw_enter(&perdm_rwlock, RW_WRITER);
2679 2684 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2680 2685 if (p->dm_str == str) { /* already present */
2681 2686 p->dm_ref++;
2682 2687 rw_exit(&perdm_rwlock);
2683 2688 free_syncq(sq);
2684 2689 kmem_free(dmp, sizeof (perdm_t));
2685 2690 return (p);
2686 2691 }
2687 2692 }
2688 2693
2689 2694 *pp = dmp;
2690 2695 rw_exit(&perdm_rwlock);
2691 2696 return (dmp);
2692 2697 }
2693 2698
2694 2699 void
2695 2700 rele_dm(perdm_t *dmp)
2696 2701 {
2697 2702 perdm_t **pp;
2698 2703 perdm_t *p;
2699 2704
2700 2705 rw_enter(&perdm_rwlock, RW_WRITER);
2701 2706 ASSERT(dmp->dm_ref > 0);
2702 2707
2703 2708 if (--dmp->dm_ref > 0) {
2704 2709 rw_exit(&perdm_rwlock);
2705 2710 return;
2706 2711 }
2707 2712
2708 2713 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2709 2714 if (p == dmp)
2710 2715 break;
2711 2716 ASSERT(p == dmp);
2712 2717 *pp = p->dm_next;
2713 2718 rw_exit(&perdm_rwlock);
2714 2719
2715 2720 /*
2716 2721 * Wait for any background processing that relies on the
2717 2722 * syncq to complete before it is freed.
2718 2723 */
2719 2724 wait_sq_svc(p->dm_sq);
2720 2725 free_syncq(p->dm_sq);
2721 2726 kmem_free(p, sizeof (perdm_t));
2722 2727 }
2723 2728
2724 2729 /*
2725 2730 * Make a protocol message given control and data buffers.
2726 2731 * n.b., this can block; be careful of what locks you hold when calling it.
2727 2732 *
2728 2733 * If sd_maxblk is less than *iosize this routine can fail part way through
2729 2734 * (due to an allocation failure). In this case on return *iosize will contain
2730 2735 * the amount that was consumed. Otherwise *iosize will not be modified
2731 2736 * i.e. it will contain the amount that was consumed.
2732 2737 */
2733 2738 int
2734 2739 strmakemsg(
2735 2740 struct strbuf *mctl,
2736 2741 ssize_t *iosize,
2737 2742 struct uio *uiop,
2738 2743 stdata_t *stp,
2739 2744 int32_t flag,
2740 2745 mblk_t **mpp)
2741 2746 {
2742 2747 mblk_t *mpctl = NULL;
2743 2748 mblk_t *mpdata = NULL;
2744 2749 int error;
2745 2750
2746 2751 ASSERT(uiop != NULL);
2747 2752
2748 2753 *mpp = NULL;
2749 2754 /* Create control part, if any */
2750 2755 if ((mctl != NULL) && (mctl->len >= 0)) {
2751 2756 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2752 2757 if (error)
2753 2758 return (error);
2754 2759 }
2755 2760 /* Create data part, if any */
2756 2761 if (*iosize >= 0) {
2757 2762 error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2758 2763 if (error) {
2759 2764 freemsg(mpctl);
2760 2765 return (error);
2761 2766 }
2762 2767 }
2763 2768 if (mpctl != NULL) {
2764 2769 if (mpdata != NULL)
2765 2770 linkb(mpctl, mpdata);
2766 2771 *mpp = mpctl;
2767 2772 } else {
2768 2773 *mpp = mpdata;
2769 2774 }
2770 2775 return (0);
2771 2776 }
2772 2777
2773 2778 /*
2774 2779 * Make the control part of a protocol message given a control buffer.
2775 2780 * n.b., this can block; be careful of what locks you hold when calling it.
2776 2781 */
2777 2782 int
2778 2783 strmakectl(
2779 2784 struct strbuf *mctl,
2780 2785 int32_t flag,
2781 2786 int32_t fflag,
2782 2787 mblk_t **mpp)
2783 2788 {
2784 2789 mblk_t *bp = NULL;
2785 2790 unsigned char msgtype;
2786 2791 int error = 0;
2787 2792 cred_t *cr = CRED();
2788 2793
2789 2794 /* We do not support interrupt threads using the stream head to send */
2790 2795 ASSERT(cr != NULL);
2791 2796
2792 2797 *mpp = NULL;
2793 2798 /*
2794 2799 * Create control part of message, if any.
2795 2800 */
2796 2801 if ((mctl != NULL) && (mctl->len >= 0)) {
2797 2802 caddr_t base;
2798 2803 int ctlcount;
2799 2804 int allocsz;
2800 2805
2801 2806 if (flag & RS_HIPRI)
2802 2807 msgtype = M_PCPROTO;
2803 2808 else
2804 2809 msgtype = M_PROTO;
2805 2810
2806 2811 ctlcount = mctl->len;
2807 2812 base = mctl->buf;
2808 2813
2809 2814 /*
2810 2815 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2811 2816 * blocks by increasing the size to something more usable.
2812 2817 */
2813 2818 allocsz = MAX(ctlcount, 64);
2814 2819
2815 2820 /*
2816 2821 * Range checking has already been done; simply try
2817 2822 * to allocate a message block for the ctl part.
2818 2823 */
2819 2824 while ((bp = allocb_cred(allocsz, cr,
2820 2825 curproc->p_pid)) == NULL) {
2821 2826 if (fflag & (FNDELAY|FNONBLOCK))
2822 2827 return (EAGAIN);
2823 2828 if (error = strwaitbuf(allocsz, BPRI_MED))
2824 2829 return (error);
2825 2830 }
2826 2831
2827 2832 bp->b_datap->db_type = msgtype;
2828 2833 if (copyin(base, bp->b_wptr, ctlcount)) {
2829 2834 freeb(bp);
2830 2835 return (EFAULT);
2831 2836 }
2832 2837 bp->b_wptr += ctlcount;
2833 2838 }
2834 2839 *mpp = bp;
2835 2840 return (0);
2836 2841 }
2837 2842
2838 2843 /*
2839 2844 * Make a protocol message given data buffers.
2840 2845 * n.b., this can block; be careful of what locks you hold when calling it.
2841 2846 *
2842 2847 * If sd_maxblk is less than *iosize this routine can fail part way through
2843 2848 * (due to an allocation failure). In this case on return *iosize will contain
2844 2849 * the amount that was consumed. Otherwise *iosize will not be modified
2845 2850 * i.e. it will contain the amount that was consumed.
2846 2851 */
2847 2852 int
2848 2853 strmakedata(
2849 2854 ssize_t *iosize,
2850 2855 struct uio *uiop,
2851 2856 stdata_t *stp,
2852 2857 int32_t flag,
2853 2858 mblk_t **mpp)
2854 2859 {
2855 2860 mblk_t *mp = NULL;
2856 2861 mblk_t *bp;
2857 2862 int wroff = (int)stp->sd_wroff;
2858 2863 int tail_len = (int)stp->sd_tail;
2859 2864 int extra = wroff + tail_len;
2860 2865 int error = 0;
2861 2866 ssize_t maxblk;
2862 2867 ssize_t count = *iosize;
2863 2868 cred_t *cr;
2864 2869
2865 2870 *mpp = NULL;
2866 2871 if (count < 0)
2867 2872 return (0);
2868 2873
2869 2874 /* We do not support interrupt threads using the stream head to send */
2870 2875 cr = CRED();
2871 2876 ASSERT(cr != NULL);
2872 2877
2873 2878 maxblk = stp->sd_maxblk;
2874 2879 if (maxblk == INFPSZ)
2875 2880 maxblk = count;
2876 2881
2877 2882 /*
2878 2883 * Create data part of message, if any.
2879 2884 */
2880 2885 do {
2881 2886 ssize_t size;
2882 2887 dblk_t *dp;
2883 2888
2884 2889 ASSERT(uiop);
2885 2890
2886 2891 size = MIN(count, maxblk);
2887 2892
2888 2893 while ((bp = allocb_cred(size + extra, cr,
2889 2894 curproc->p_pid)) == NULL) {
2890 2895 error = EAGAIN;
2891 2896 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2892 2897 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2893 2898 if (count == *iosize) {
2894 2899 freemsg(mp);
2895 2900 return (error);
2896 2901 } else {
2897 2902 *iosize -= count;
2898 2903 *mpp = mp;
2899 2904 return (0);
2900 2905 }
2901 2906 }
2902 2907 }
2903 2908 dp = bp->b_datap;
2904 2909 dp->db_cpid = curproc->p_pid;
2905 2910 ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2906 2911 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2907 2912
2908 2913 if (flag & STRUIO_POSTPONE) {
2909 2914 /*
2910 2915 * Setup the stream uio portion of the
2911 2916 * dblk for subsequent use by struioget().
2912 2917 */
2913 2918 dp->db_struioflag = STRUIO_SPEC;
2914 2919 dp->db_cksumstart = 0;
2915 2920 dp->db_cksumstuff = 0;
2916 2921 dp->db_cksumend = size;
2917 2922 *(long long *)dp->db_struioun.data = 0ll;
2918 2923 bp->b_wptr += size;
2919 2924 } else {
2920 2925 if (stp->sd_copyflag & STRCOPYCACHED)
2921 2926 uiop->uio_extflg |= UIO_COPY_CACHED;
2922 2927
2923 2928 if (size != 0) {
2924 2929 error = uiomove(bp->b_wptr, size, UIO_WRITE,
2925 2930 uiop);
2926 2931 if (error != 0) {
2927 2932 freeb(bp);
2928 2933 freemsg(mp);
2929 2934 return (error);
2930 2935 }
2931 2936 }
2932 2937 bp->b_wptr += size;
2933 2938
2934 2939 if (stp->sd_wputdatafunc != NULL) {
2935 2940 mblk_t *newbp;
2936 2941
2937 2942 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2938 2943 bp, NULL, NULL, NULL, NULL);
2939 2944 if (newbp == NULL) {
2940 2945 freeb(bp);
2941 2946 freemsg(mp);
2942 2947 return (ECOMM);
2943 2948 }
2944 2949 bp = newbp;
2945 2950 }
2946 2951 }
2947 2952
2948 2953 count -= size;
2949 2954
2950 2955 if (mp == NULL)
2951 2956 mp = bp;
2952 2957 else
2953 2958 linkb(mp, bp);
2954 2959 } while (count > 0);
2955 2960
2956 2961 *mpp = mp;
2957 2962 return (0);
2958 2963 }
2959 2964
2960 2965 /*
2961 2966 * Wait for a buffer to become available. Return non-zero errno
2962 2967 * if not able to wait, 0 if buffer is probably there.
2963 2968 */
2964 2969 int
2965 2970 strwaitbuf(size_t size, int pri)
2966 2971 {
2967 2972 bufcall_id_t id;
2968 2973
2969 2974 mutex_enter(&bcall_monitor);
2970 2975 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2971 2976 &ttoproc(curthread)->p_flag_cv)) == 0) {
2972 2977 mutex_exit(&bcall_monitor);
2973 2978 return (ENOSR);
2974 2979 }
2975 2980 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2976 2981 unbufcall(id);
2977 2982 mutex_exit(&bcall_monitor);
2978 2983 return (EINTR);
2979 2984 }
2980 2985 unbufcall(id);
2981 2986 mutex_exit(&bcall_monitor);
2982 2987 return (0);
2983 2988 }
2984 2989
2985 2990 /*
2986 2991 * This function waits for a read or write event to happen on a stream.
2987 2992 * fmode can specify FNDELAY and/or FNONBLOCK.
2988 2993 * The timeout is in ms with -1 meaning infinite.
2989 2994 * The flag values work as follows:
2990 2995 * READWAIT Check for read side errors, send M_READ
2991 2996 * GETWAIT Check for read side errors, no M_READ
2992 2997 * WRITEWAIT Check for write side errors.
2993 2998 * NOINTR Do not return error if nonblocking or timeout.
2994 2999 * STR_NOERROR Ignore all errors except STPLEX.
2995 3000 * STR_NOSIG Ignore/hold signals during the duration of the call.
2996 3001 * STR_PEEK Pass through the strgeterr().
2997 3002 */
2998 3003 int
2999 3004 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3000 3005 int *done)
3001 3006 {
3002 3007 int slpflg, errs;
3003 3008 int error;
3004 3009 kcondvar_t *sleepon;
3005 3010 mblk_t *mp;
3006 3011 ssize_t *rd_count;
3007 3012 clock_t rval;
3008 3013
3009 3014 ASSERT(MUTEX_HELD(&stp->sd_lock));
3010 3015 if ((flag & READWAIT) || (flag & GETWAIT)) {
3011 3016 slpflg = RSLEEP;
3012 3017 sleepon = &_RD(stp->sd_wrq)->q_wait;
3013 3018 errs = STRDERR|STPLEX;
3014 3019 } else {
3015 3020 slpflg = WSLEEP;
3016 3021 sleepon = &stp->sd_wrq->q_wait;
3017 3022 errs = STWRERR|STRHUP|STPLEX;
3018 3023 }
3019 3024 if (flag & STR_NOERROR)
3020 3025 errs = STPLEX;
3021 3026
3022 3027 if (stp->sd_wakeq & slpflg) {
3023 3028 /*
3024 3029 * A strwakeq() is pending, no need to sleep.
3025 3030 */
3026 3031 stp->sd_wakeq &= ~slpflg;
3027 3032 *done = 0;
3028 3033 return (0);
3029 3034 }
3030 3035
3031 3036 if (stp->sd_flag & errs) {
3032 3037 /*
3033 3038 * Check for errors before going to sleep since the
3034 3039 * caller might not have checked this while holding
3035 3040 * sd_lock.
3036 3041 */
3037 3042 error = strgeterr(stp, errs, (flag & STR_PEEK));
3038 3043 if (error != 0) {
3039 3044 *done = 1;
3040 3045 return (error);
3041 3046 }
3042 3047 }
3043 3048
3044 3049 /*
3045 3050 * If any module downstream has requested read notification
3046 3051 * by setting SNDMREAD flag using M_SETOPTS, send a message
3047 3052 * down stream.
3048 3053 */
3049 3054 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3050 3055 mutex_exit(&stp->sd_lock);
3051 3056 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3052 3057 (flag & STR_NOSIG), &error))) {
3053 3058 mutex_enter(&stp->sd_lock);
3054 3059 *done = 1;
3055 3060 return (error);
3056 3061 }
3057 3062 mp->b_datap->db_type = M_READ;
3058 3063 rd_count = (ssize_t *)mp->b_wptr;
3059 3064 *rd_count = count;
3060 3065 mp->b_wptr += sizeof (ssize_t);
3061 3066 /*
3062 3067 * Send the number of bytes requested by the
3063 3068 * read as the argument to M_READ.
3064 3069 */
3065 3070 stream_willservice(stp);
3066 3071 putnext(stp->sd_wrq, mp);
3067 3072 stream_runservice(stp);
3068 3073 mutex_enter(&stp->sd_lock);
3069 3074
3070 3075 /*
3071 3076 * If any data arrived due to inline processing
3072 3077 * of putnext(), don't sleep.
3073 3078 */
3074 3079 if (_RD(stp->sd_wrq)->q_first != NULL) {
3075 3080 *done = 0;
3076 3081 return (0);
3077 3082 }
3078 3083 }
3079 3084
3080 3085 if (fmode & (FNDELAY|FNONBLOCK)) {
3081 3086 if (!(flag & NOINTR))
3082 3087 error = EAGAIN;
3083 3088 else
3084 3089 error = 0;
3085 3090 *done = 1;
3086 3091 return (error);
3087 3092 }
3088 3093
3089 3094 stp->sd_flag |= slpflg;
3090 3095 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3091 3096 "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3092 3097 stp, flag, count, fmode, done);
3093 3098
3094 3099 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3095 3100 if (rval > 0) {
3096 3101 /* EMPTY */
3097 3102 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3098 3103 "strwaitq awakes(2):%X, %X, %X, %X, %X",
3099 3104 stp, flag, count, fmode, done);
3100 3105 } else if (rval == 0) {
3101 3106 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3102 3107 "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3103 3108 stp, flag, count, fmode, done);
3104 3109 stp->sd_flag &= ~slpflg;
3105 3110 cv_broadcast(sleepon);
3106 3111 if (!(flag & NOINTR))
3107 3112 error = EINTR;
3108 3113 else
3109 3114 error = 0;
3110 3115 *done = 1;
3111 3116 return (error);
3112 3117 } else {
3113 3118 /* timeout */
3114 3119 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3115 3120 "strwaitq timeout:%p, %X, %lX, %X, %p",
3116 3121 stp, flag, count, fmode, done);
3117 3122 *done = 1;
3118 3123 if (!(flag & NOINTR))
3119 3124 return (ETIME);
3120 3125 else
3121 3126 return (0);
3122 3127 }
3123 3128 /*
3124 3129 * If the caller implements delayed errors (i.e. queued after data)
3125 3130 * we can not check for errors here since data as well as an
3126 3131 * error might have arrived at the stream head. We return to
3127 3132 * have the caller check the read queue before checking for errors.
3128 3133 */
3129 3134 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3130 3135 error = strgeterr(stp, errs, (flag & STR_PEEK));
3131 3136 if (error != 0) {
3132 3137 *done = 1;
3133 3138 return (error);
3134 3139 }
3135 3140 }
3136 3141 *done = 0;
3137 3142 return (0);
3138 3143 }
3139 3144
3140 3145 /*
3141 3146 * Perform job control discipline access checks.
3142 3147 * Return 0 for success and the errno for failure.
3143 3148 */
3144 3149
3145 3150 #define cantsend(p, t, sig) \
3146 3151 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3147 3152
3148 3153 int
3149 3154 straccess(struct stdata *stp, enum jcaccess mode)
3150 3155 {
3151 3156 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */
3152 3157 kthread_t *t = curthread;
3153 3158 proc_t *p = ttoproc(t);
3154 3159 sess_t *sp;
3155 3160
3156 3161 ASSERT(mutex_owned(&stp->sd_lock));
3157 3162
3158 3163 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3159 3164 return (0);
3160 3165
3161 3166 mutex_enter(&p->p_lock); /* protects p_pgidp */
3162 3167
3163 3168 for (;;) {
3164 3169 mutex_enter(&p->p_splock); /* protects p->p_sessp */
3165 3170 sp = p->p_sessp;
3166 3171 mutex_enter(&sp->s_lock); /* protects sp->* */
3167 3172
3168 3173 /*
3169 3174 * If this is not the calling process's controlling terminal
3170 3175 * or if the calling process is already in the foreground
3171 3176 * then allow access.
3172 3177 */
3173 3178 if (sp->s_dev != stp->sd_vnode->v_rdev ||
3174 3179 p->p_pgidp == stp->sd_pgidp) {
3175 3180 mutex_exit(&sp->s_lock);
3176 3181 mutex_exit(&p->p_splock);
3177 3182 mutex_exit(&p->p_lock);
3178 3183 return (0);
3179 3184 }
3180 3185
3181 3186 /*
3182 3187 * Check to see if controlling terminal has been deallocated.
3183 3188 */
3184 3189 if (sp->s_vp == NULL) {
3185 3190 if (!cantsend(p, t, SIGHUP))
3186 3191 sigtoproc(p, t, SIGHUP);
3187 3192 mutex_exit(&sp->s_lock);
3188 3193 mutex_exit(&p->p_splock);
3189 3194 mutex_exit(&p->p_lock);
3190 3195 return (EIO);
3191 3196 }
3192 3197
3193 3198 mutex_exit(&sp->s_lock);
3194 3199 mutex_exit(&p->p_splock);
3195 3200
3196 3201 if (mode == JCGETP) {
3197 3202 mutex_exit(&p->p_lock);
3198 3203 return (0);
3199 3204 }
3200 3205
3201 3206 if (mode == JCREAD) {
3202 3207 if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3203 3208 mutex_exit(&p->p_lock);
3204 3209 return (EIO);
3205 3210 }
3206 3211 mutex_exit(&p->p_lock);
3207 3212 mutex_exit(&stp->sd_lock);
3208 3213 pgsignal(p->p_pgidp, SIGTTIN);
3209 3214 mutex_enter(&stp->sd_lock);
3210 3215 mutex_enter(&p->p_lock);
3211 3216 } else { /* mode == JCWRITE or JCSETP */
3212 3217 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3213 3218 cantsend(p, t, SIGTTOU)) {
3214 3219 mutex_exit(&p->p_lock);
3215 3220 return (0);
3216 3221 }
3217 3222 if (p->p_detached) {
3218 3223 mutex_exit(&p->p_lock);
3219 3224 return (EIO);
3220 3225 }
3221 3226 mutex_exit(&p->p_lock);
3222 3227 mutex_exit(&stp->sd_lock);
3223 3228 pgsignal(p->p_pgidp, SIGTTOU);
3224 3229 mutex_enter(&stp->sd_lock);
3225 3230 mutex_enter(&p->p_lock);
3226 3231 }
3227 3232
3228 3233 /*
3229 3234 * We call cv_wait_sig_swap() to cause the appropriate
3230 3235 * action for the jobcontrol signal to take place.
3231 3236 * If the signal is being caught, we will take the
3232 3237 * EINTR error return. Otherwise, the default action
3233 3238 * of causing the process to stop will take place.
3234 3239 * In this case, we rely on the periodic cv_broadcast() on
3235 3240 * &lbolt_cv to wake us up to loop around and test again.
3236 3241 * We can't get here if the signal is ignored or
3237 3242 * if the current thread is blocking the signal.
3238 3243 */
3239 3244 mutex_exit(&stp->sd_lock);
3240 3245 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3241 3246 mutex_exit(&p->p_lock);
3242 3247 mutex_enter(&stp->sd_lock);
3243 3248 return (EINTR);
3244 3249 }
3245 3250 mutex_exit(&p->p_lock);
3246 3251 mutex_enter(&stp->sd_lock);
3247 3252 mutex_enter(&p->p_lock);
3248 3253 }
3249 3254 }
3250 3255
3251 3256 /*
3252 3257 * Return size of message of block type (bp->b_datap->db_type)
3253 3258 */
3254 3259 size_t
3255 3260 xmsgsize(mblk_t *bp)
3256 3261 {
3257 3262 unsigned char type;
3258 3263 size_t count = 0;
3259 3264
3260 3265 type = bp->b_datap->db_type;
3261 3266
3262 3267 for (; bp; bp = bp->b_cont) {
3263 3268 if (type != bp->b_datap->db_type)
3264 3269 break;
3265 3270 ASSERT(bp->b_wptr >= bp->b_rptr);
3266 3271 count += bp->b_wptr - bp->b_rptr;
3267 3272 }
3268 3273 return (count);
3269 3274 }
3270 3275
3271 3276 /*
3272 3277 * Allocate a stream head.
3273 3278 */
3274 3279 struct stdata *
3275 3280 shalloc(queue_t *qp)
3276 3281 {
3277 3282 stdata_t *stp;
3278 3283
3279 3284 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3280 3285
3281 3286 stp->sd_wrq = _WR(qp);
3282 3287 stp->sd_strtab = NULL;
3283 3288 stp->sd_iocid = 0;
3284 3289 stp->sd_mate = NULL;
3285 3290 stp->sd_freezer = NULL;
3286 3291 stp->sd_refcnt = 0;
3287 3292 stp->sd_wakeq = 0;
3288 3293 stp->sd_anchor = 0;
3289 3294 stp->sd_struiowrq = NULL;
3290 3295 stp->sd_struiordq = NULL;
3291 3296 stp->sd_struiodnak = 0;
3292 3297 stp->sd_struionak = NULL;
3293 3298 stp->sd_t_audit_data = NULL;
3294 3299 stp->sd_rput_opt = 0;
3295 3300 stp->sd_wput_opt = 0;
3296 3301 stp->sd_read_opt = 0;
3297 3302 stp->sd_rprotofunc = strrput_proto;
3298 3303 stp->sd_rmiscfunc = strrput_misc;
3299 3304 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3300 3305 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3301 3306 stp->sd_ciputctrl = NULL;
3302 3307 stp->sd_nciputctrl = 0;
3303 3308 stp->sd_qhead = NULL;
3304 3309 stp->sd_qtail = NULL;
3305 3310 stp->sd_servid = NULL;
3306 3311 stp->sd_nqueues = 0;
3307 3312 stp->sd_svcflags = 0;
3308 3313 stp->sd_copyflag = 0;
↓ open down ↓ |
946 lines elided |
↑ open up ↑ |
3309 3314
3310 3315 return (stp);
3311 3316 }
3312 3317
3313 3318 /*
3314 3319 * Free a stream head.
3315 3320 */
3316 3321 void
3317 3322 shfree(stdata_t *stp)
3318 3323 {
3324 + pid_node_t *pn;
3325 +
3319 3326 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3320 3327
3321 3328 stp->sd_wrq = NULL;
3322 3329
3323 3330 mutex_enter(&stp->sd_qlock);
3324 3331 while (stp->sd_svcflags & STRS_SCHEDULED) {
3325 3332 STRSTAT(strwaits);
3326 3333 cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3327 3334 }
3328 3335 mutex_exit(&stp->sd_qlock);
3329 3336
3330 3337 if (stp->sd_ciputctrl != NULL) {
3331 3338 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
3332 3339 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3333 3340 stp->sd_nciputctrl, 0);
3334 3341 ASSERT(ciputctrl_cache != NULL);
3335 3342 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3336 3343 stp->sd_ciputctrl = NULL;
3337 3344 stp->sd_nciputctrl = 0;
3338 3345 }
3339 3346 ASSERT(stp->sd_qhead == NULL);
3340 3347 ASSERT(stp->sd_qtail == NULL);
3341 3348 ASSERT(stp->sd_nqueues == 0);
3349 +
3350 + mutex_enter(&stp->sd_pid_tree_lock);
3351 + while ((pn = avl_first(&stp->sd_pid_tree)) != NULL) {
3352 + avl_remove(&stp->sd_pid_tree, pn);
3353 + kmem_free(pn, sizeof (*pn));
3354 + }
3355 + mutex_exit(&stp->sd_pid_tree_lock);
3356 +
3342 3357 kmem_cache_free(stream_head_cache, stp);
3343 3358 }
3344 3359
3360 +void
3361 +sh_insert_pid(struct stdata *stp, pid_t pid)
3362 +{
3363 + pid_node_t *pn, lookup_pn;
3364 + avl_index_t idx_pn;
3365 +
3366 + lookup_pn.pn_pid = pid;
3367 + mutex_enter(&stp->sd_pid_tree_lock);
3368 + pn = avl_find(&stp->sd_pid_tree, &lookup_pn, &idx_pn);
3369 +
3370 + if (pn != NULL) {
3371 + pn->pn_count++;
3372 + } else {
3373 + pn = kmem_zalloc(sizeof (*pn), KM_SLEEP);
3374 + pn->pn_pid = pid;
3375 + pn->pn_count = 1;
3376 + avl_insert(&stp->sd_pid_tree, pn, idx_pn);
3377 + }
3378 + mutex_exit(&stp->sd_pid_tree_lock);
3379 +}
3380 +
3381 +void
3382 +sh_remove_pid(struct stdata *stp, pid_t pid)
3383 +{
3384 + pid_node_t *pn, lookup_pn;
3385 +
3386 + lookup_pn.pn_pid = pid;
3387 + mutex_enter(&stp->sd_pid_tree_lock);
3388 + pn = avl_find(&stp->sd_pid_tree, &lookup_pn, NULL);
3389 +
3390 + if (pn != NULL) {
3391 + if (pn->pn_count > 1) {
3392 + pn->pn_count--;
3393 + } else {
3394 + avl_remove(&stp->sd_pid_tree, pn);
3395 + kmem_free(pn, sizeof (*pn));
3396 + }
3397 + }
3398 + mutex_exit(&stp->sd_pid_tree_lock);
3399 +}
3400 +
3401 +mblk_t *
3402 +sh_get_pid_mblk(struct stdata *stp)
3403 +{
3404 + mblk_t *mblk;
3405 + ulong_t sz, n;
3406 + pid_t *pids;
3407 + pid_node_t *pn;
3408 + conn_pid_info_t *cpi;
3409 +
3410 + mutex_enter(&stp->sd_pid_tree_lock);
3411 +
3412 + n = avl_numnodes(&stp->sd_pid_tree);
3413 + sz = sizeof (conn_pid_info_t);
3414 + sz += (n > 1) ? ((n - 1) * sizeof (pid_t)) : 0;
3415 + if ((mblk = allocb(sz, BPRI_HI)) == NULL) {
3416 + mutex_exit(&stp->sd_pid_tree_lock);
3417 + return (NULL);
3418 + }
3419 + mblk->b_wptr += sz;
3420 + cpi = (conn_pid_info_t *)mblk->b_datap->db_base;
3421 + cpi->cpi_magic = CONN_PID_INFO_MGC;
3422 + cpi->cpi_contents = CONN_PID_INFO_XTI;
3423 + cpi->cpi_pids_cnt = n;
3424 + cpi->cpi_tot_size = sz;
3425 + cpi->cpi_pids[0] = 0;
3426 +
3427 + if (cpi->cpi_pids_cnt > 0) {
3428 + pids = cpi->cpi_pids;
3429 + for (pn = avl_first(&stp->sd_pid_tree); pn != NULL;
3430 + pids++, pn = AVL_NEXT(&stp->sd_pid_tree, pn))
3431 + *pids = pn->pn_pid;
3432 + }
3433 + mutex_exit(&stp->sd_pid_tree_lock);
3434 + return (mblk);
3435 +}
3436 +
3345 3437 /*
3346 3438 * Allocate a pair of queues and a syncq for the pair
3347 3439 */
3348 3440 queue_t *
3349 3441 allocq(void)
3350 3442 {
3351 3443 queinfo_t *qip;
3352 3444 queue_t *qp, *wqp;
3353 3445 syncq_t *sq;
3354 3446
3355 3447 qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3356 3448
3357 3449 qp = &qip->qu_rqueue;
3358 3450 wqp = &qip->qu_wqueue;
3359 3451 sq = &qip->qu_syncq;
3360 3452
3361 3453 qp->q_last = NULL;
3362 3454 qp->q_next = NULL;
3363 3455 qp->q_ptr = NULL;
3364 3456 qp->q_flag = QUSE | QREADR;
3365 3457 qp->q_bandp = NULL;
3366 3458 qp->q_stream = NULL;
3367 3459 qp->q_syncq = sq;
3368 3460 qp->q_nband = 0;
3369 3461 qp->q_nfsrv = NULL;
3370 3462 qp->q_draining = 0;
3371 3463 qp->q_syncqmsgs = 0;
3372 3464 qp->q_spri = 0;
3373 3465 qp->q_qtstamp = 0;
3374 3466 qp->q_sqtstamp = 0;
3375 3467 qp->q_fp = NULL;
3376 3468
3377 3469 wqp->q_last = NULL;
3378 3470 wqp->q_next = NULL;
3379 3471 wqp->q_ptr = NULL;
3380 3472 wqp->q_flag = QUSE;
3381 3473 wqp->q_bandp = NULL;
3382 3474 wqp->q_stream = NULL;
3383 3475 wqp->q_syncq = sq;
3384 3476 wqp->q_nband = 0;
3385 3477 wqp->q_nfsrv = NULL;
3386 3478 wqp->q_draining = 0;
3387 3479 wqp->q_syncqmsgs = 0;
3388 3480 wqp->q_qtstamp = 0;
3389 3481 wqp->q_sqtstamp = 0;
3390 3482 wqp->q_spri = 0;
3391 3483
3392 3484 sq->sq_count = 0;
3393 3485 sq->sq_rmqcount = 0;
3394 3486 sq->sq_flags = 0;
3395 3487 sq->sq_type = 0;
3396 3488 sq->sq_callbflags = 0;
3397 3489 sq->sq_cancelid = 0;
3398 3490 sq->sq_ciputctrl = NULL;
3399 3491 sq->sq_nciputctrl = 0;
3400 3492 sq->sq_needexcl = 0;
3401 3493 sq->sq_svcflags = 0;
3402 3494
3403 3495 return (qp);
3404 3496 }
3405 3497
3406 3498 /*
3407 3499 * Free a pair of queues and the "attached" syncq.
3408 3500 * Discard any messages left on the syncq(s), remove the syncq(s) from the
3409 3501 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3410 3502 */
3411 3503 void
3412 3504 freeq(queue_t *qp)
3413 3505 {
3414 3506 qband_t *qbp, *nqbp;
3415 3507 syncq_t *sq, *outer;
3416 3508 queue_t *wqp = _WR(qp);
3417 3509
3418 3510 ASSERT(qp->q_flag & QREADR);
3419 3511
3420 3512 /*
3421 3513 * If a previously dispatched taskq job is scheduled to run
3422 3514 * sync_service() or a service routine is scheduled for the
3423 3515 * queues about to be freed, wait here until all service is
3424 3516 * done on the queue and all associated queues and syncqs.
3425 3517 */
3426 3518 wait_svc(qp);
3427 3519
3428 3520 (void) flush_syncq(qp->q_syncq, qp);
3429 3521 (void) flush_syncq(wqp->q_syncq, wqp);
3430 3522 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3431 3523
3432 3524 /*
3433 3525 * Flush the queues before q_next is set to NULL This is needed
3434 3526 * in order to backenable any downstream queue before we go away.
3435 3527 * Note: we are already removed from the stream so that the
3436 3528 * backenabling will not cause any messages to be delivered to our
3437 3529 * put procedures.
3438 3530 */
3439 3531 flushq(qp, FLUSHALL);
3440 3532 flushq(wqp, FLUSHALL);
3441 3533
3442 3534 /* Tidy up - removeq only does a half-remove from stream */
3443 3535 qp->q_next = wqp->q_next = NULL;
3444 3536 ASSERT(!(qp->q_flag & QENAB));
3445 3537 ASSERT(!(wqp->q_flag & QENAB));
3446 3538
3447 3539 outer = qp->q_syncq->sq_outer;
3448 3540 if (outer != NULL) {
3449 3541 outer_remove(outer, qp->q_syncq);
3450 3542 if (wqp->q_syncq != qp->q_syncq)
3451 3543 outer_remove(outer, wqp->q_syncq);
3452 3544 }
3453 3545 /*
3454 3546 * Free any syncqs that are outside what allocq returned.
3455 3547 */
3456 3548 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3457 3549 free_syncq(qp->q_syncq);
3458 3550 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3459 3551 free_syncq(wqp->q_syncq);
3460 3552
3461 3553 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3462 3554 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3463 3555 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3464 3556 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3465 3557 sq = SQ(qp);
3466 3558 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3467 3559 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3468 3560 ASSERT(sq->sq_outer == NULL);
3469 3561 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3470 3562 ASSERT(sq->sq_callbpend == NULL);
3471 3563 ASSERT(sq->sq_needexcl == 0);
3472 3564
3473 3565 if (sq->sq_ciputctrl != NULL) {
3474 3566 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3475 3567 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3476 3568 sq->sq_nciputctrl, 0);
3477 3569 ASSERT(ciputctrl_cache != NULL);
3478 3570 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3479 3571 sq->sq_ciputctrl = NULL;
3480 3572 sq->sq_nciputctrl = 0;
3481 3573 }
3482 3574
3483 3575 ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3484 3576 ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3485 3577 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3486 3578
3487 3579 qp->q_flag &= ~QUSE;
3488 3580 wqp->q_flag &= ~QUSE;
3489 3581
3490 3582 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3491 3583 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3492 3584
3493 3585 qbp = qp->q_bandp;
3494 3586 while (qbp) {
3495 3587 nqbp = qbp->qb_next;
3496 3588 freeband(qbp);
3497 3589 qbp = nqbp;
3498 3590 }
3499 3591 qbp = wqp->q_bandp;
3500 3592 while (qbp) {
3501 3593 nqbp = qbp->qb_next;
3502 3594 freeband(qbp);
3503 3595 qbp = nqbp;
3504 3596 }
3505 3597 kmem_cache_free(queue_cache, qp);
3506 3598 }
3507 3599
3508 3600 /*
3509 3601 * Allocate a qband structure.
3510 3602 */
3511 3603 qband_t *
3512 3604 allocband(void)
3513 3605 {
3514 3606 qband_t *qbp;
3515 3607
3516 3608 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3517 3609 if (qbp == NULL)
3518 3610 return (NULL);
3519 3611
3520 3612 qbp->qb_next = NULL;
3521 3613 qbp->qb_count = 0;
3522 3614 qbp->qb_mblkcnt = 0;
3523 3615 qbp->qb_first = NULL;
3524 3616 qbp->qb_last = NULL;
3525 3617 qbp->qb_flag = 0;
3526 3618
3527 3619 return (qbp);
3528 3620 }
3529 3621
3530 3622 /*
3531 3623 * Free a qband structure.
3532 3624 */
3533 3625 void
3534 3626 freeband(qband_t *qbp)
3535 3627 {
3536 3628 kmem_cache_free(qband_cache, qbp);
3537 3629 }
3538 3630
3539 3631 /*
3540 3632 * Just like putnextctl(9F), except that allocb_wait() is used.
3541 3633 *
3542 3634 * Consolidation Private, and of course only callable from the stream head or
3543 3635 * routines that may block.
3544 3636 */
3545 3637 int
3546 3638 putnextctl_wait(queue_t *q, int type)
3547 3639 {
3548 3640 mblk_t *bp;
3549 3641 int error;
3550 3642
3551 3643 if ((datamsg(type) && (type != M_DELAY)) ||
3552 3644 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3553 3645 return (0);
3554 3646
3555 3647 bp->b_datap->db_type = (unsigned char)type;
3556 3648 putnext(q, bp);
3557 3649 return (1);
3558 3650 }
3559 3651
3560 3652 /*
3561 3653 * Run any possible bufcalls.
3562 3654 */
3563 3655 void
3564 3656 runbufcalls(void)
3565 3657 {
3566 3658 strbufcall_t *bcp;
3567 3659
3568 3660 mutex_enter(&bcall_monitor);
3569 3661 mutex_enter(&strbcall_lock);
3570 3662
3571 3663 if (strbcalls.bc_head) {
3572 3664 size_t count;
3573 3665 int nevent;
3574 3666
3575 3667 /*
3576 3668 * count how many events are on the list
3577 3669 * now so we can check to avoid looping
3578 3670 * in low memory situations
3579 3671 */
3580 3672 nevent = 0;
3581 3673 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3582 3674 nevent++;
3583 3675
3584 3676 /*
3585 3677 * get estimate of available memory from kmem_avail().
3586 3678 * awake all bufcall functions waiting for
3587 3679 * memory whose request could be satisfied
3588 3680 * by 'count' memory and let 'em fight for it.
3589 3681 */
3590 3682 count = kmem_avail();
3591 3683 while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3592 3684 STRSTAT(bufcalls);
3593 3685 --nevent;
3594 3686 if (bcp->bc_size <= count) {
3595 3687 bcp->bc_executor = curthread;
3596 3688 mutex_exit(&strbcall_lock);
3597 3689 (*bcp->bc_func)(bcp->bc_arg);
3598 3690 mutex_enter(&strbcall_lock);
3599 3691 bcp->bc_executor = NULL;
3600 3692 cv_broadcast(&bcall_cv);
3601 3693 strbcalls.bc_head = bcp->bc_next;
3602 3694 kmem_free(bcp, sizeof (strbufcall_t));
3603 3695 } else {
3604 3696 /*
3605 3697 * too big, try again later - note
3606 3698 * that nevent was decremented above
3607 3699 * so we won't retry this one on this
3608 3700 * iteration of the loop
3609 3701 */
3610 3702 if (bcp->bc_next != NULL) {
3611 3703 strbcalls.bc_head = bcp->bc_next;
3612 3704 bcp->bc_next = NULL;
3613 3705 strbcalls.bc_tail->bc_next = bcp;
3614 3706 strbcalls.bc_tail = bcp;
3615 3707 }
3616 3708 }
3617 3709 }
3618 3710 if (strbcalls.bc_head == NULL)
3619 3711 strbcalls.bc_tail = NULL;
3620 3712 }
3621 3713
3622 3714 mutex_exit(&strbcall_lock);
3623 3715 mutex_exit(&bcall_monitor);
3624 3716 }
3625 3717
3626 3718
3627 3719 /*
3628 3720 * Actually run queue's service routine.
3629 3721 */
3630 3722 static void
3631 3723 runservice(queue_t *q)
3632 3724 {
3633 3725 qband_t *qbp;
3634 3726
3635 3727 ASSERT(q->q_qinfo->qi_srvp);
3636 3728 again:
3637 3729 entersq(q->q_syncq, SQ_SVC);
3638 3730 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3639 3731 "runservice starts:%p", q);
3640 3732
3641 3733 if (!(q->q_flag & QWCLOSE))
3642 3734 (*q->q_qinfo->qi_srvp)(q);
3643 3735
3644 3736 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3645 3737 "runservice ends:(%p)", q);
3646 3738
3647 3739 leavesq(q->q_syncq, SQ_SVC);
3648 3740
3649 3741 mutex_enter(QLOCK(q));
3650 3742 if (q->q_flag & QENAB) {
3651 3743 q->q_flag &= ~QENAB;
3652 3744 mutex_exit(QLOCK(q));
3653 3745 goto again;
3654 3746 }
3655 3747 q->q_flag &= ~QINSERVICE;
3656 3748 q->q_flag &= ~QBACK;
3657 3749 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3658 3750 qbp->qb_flag &= ~QB_BACK;
3659 3751 /*
3660 3752 * Wakeup thread waiting for the service procedure
3661 3753 * to be run (strclose and qdetach).
3662 3754 */
3663 3755 cv_broadcast(&q->q_wait);
3664 3756
3665 3757 mutex_exit(QLOCK(q));
3666 3758 }
3667 3759
3668 3760 /*
3669 3761 * Background processing of bufcalls.
3670 3762 */
3671 3763 void
3672 3764 streams_bufcall_service(void)
3673 3765 {
3674 3766 callb_cpr_t cprinfo;
3675 3767
3676 3768 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3677 3769 "streams_bufcall_service");
3678 3770
3679 3771 mutex_enter(&strbcall_lock);
3680 3772
3681 3773 for (;;) {
3682 3774 if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3683 3775 mutex_exit(&strbcall_lock);
3684 3776 runbufcalls();
3685 3777 mutex_enter(&strbcall_lock);
3686 3778 }
3687 3779 if (strbcalls.bc_head != NULL) {
3688 3780 STRSTAT(bcwaits);
3689 3781 /* Wait for memory to become available */
3690 3782 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3691 3783 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3692 3784 SEC_TO_TICK(60), TR_CLOCK_TICK);
3693 3785 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3694 3786 }
3695 3787
3696 3788 /* Wait for new work to arrive */
3697 3789 if (strbcalls.bc_head == NULL) {
3698 3790 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3699 3791 cv_wait(&strbcall_cv, &strbcall_lock);
3700 3792 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3701 3793 }
3702 3794 }
3703 3795 }
3704 3796
3705 3797 /*
3706 3798 * Background processing of streams background tasks which failed
3707 3799 * taskq_dispatch.
3708 3800 */
3709 3801 static void
3710 3802 streams_qbkgrnd_service(void)
3711 3803 {
3712 3804 callb_cpr_t cprinfo;
3713 3805 queue_t *q;
3714 3806
3715 3807 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3716 3808 "streams_bkgrnd_service");
3717 3809
3718 3810 mutex_enter(&service_queue);
3719 3811
3720 3812 for (;;) {
3721 3813 /*
3722 3814 * Wait for work to arrive.
3723 3815 */
3724 3816 while ((freebs_list == NULL) && (qhead == NULL)) {
3725 3817 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3726 3818 cv_wait(&services_to_run, &service_queue);
3727 3819 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3728 3820 }
3729 3821 /*
3730 3822 * Handle all pending freebs requests to free memory.
3731 3823 */
3732 3824 while (freebs_list != NULL) {
3733 3825 mblk_t *mp = freebs_list;
3734 3826 freebs_list = mp->b_next;
3735 3827 mutex_exit(&service_queue);
3736 3828 mblk_free(mp);
3737 3829 mutex_enter(&service_queue);
3738 3830 }
3739 3831 /*
3740 3832 * Run pending queues.
3741 3833 */
3742 3834 while (qhead != NULL) {
3743 3835 DQ(q, qhead, qtail, q_link);
3744 3836 ASSERT(q != NULL);
3745 3837 mutex_exit(&service_queue);
3746 3838 queue_service(q);
3747 3839 mutex_enter(&service_queue);
3748 3840 }
3749 3841 ASSERT(qhead == NULL && qtail == NULL);
3750 3842 }
3751 3843 }
3752 3844
3753 3845 /*
3754 3846 * Background processing of streams background tasks which failed
3755 3847 * taskq_dispatch.
3756 3848 */
3757 3849 static void
3758 3850 streams_sqbkgrnd_service(void)
3759 3851 {
3760 3852 callb_cpr_t cprinfo;
3761 3853 syncq_t *sq;
3762 3854
3763 3855 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3764 3856 "streams_sqbkgrnd_service");
3765 3857
3766 3858 mutex_enter(&service_queue);
3767 3859
3768 3860 for (;;) {
3769 3861 /*
3770 3862 * Wait for work to arrive.
3771 3863 */
3772 3864 while (sqhead == NULL) {
3773 3865 CALLB_CPR_SAFE_BEGIN(&cprinfo);
3774 3866 cv_wait(&syncqs_to_run, &service_queue);
3775 3867 CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3776 3868 }
3777 3869
3778 3870 /*
3779 3871 * Run pending syncqs.
3780 3872 */
3781 3873 while (sqhead != NULL) {
3782 3874 DQ(sq, sqhead, sqtail, sq_next);
3783 3875 ASSERT(sq != NULL);
3784 3876 ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3785 3877 mutex_exit(&service_queue);
3786 3878 syncq_service(sq);
3787 3879 mutex_enter(&service_queue);
3788 3880 }
3789 3881 }
3790 3882 }
3791 3883
3792 3884 /*
3793 3885 * Disable the syncq and wait for background syncq processing to complete.
3794 3886 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3795 3887 * list.
3796 3888 */
3797 3889 void
3798 3890 wait_sq_svc(syncq_t *sq)
3799 3891 {
3800 3892 mutex_enter(SQLOCK(sq));
3801 3893 sq->sq_svcflags |= SQ_DISABLED;
3802 3894 if (sq->sq_svcflags & SQ_BGTHREAD) {
3803 3895 syncq_t *sq_chase;
3804 3896 syncq_t *sq_curr;
3805 3897 int removed;
3806 3898
3807 3899 ASSERT(sq->sq_servcount == 1);
3808 3900 mutex_enter(&service_queue);
3809 3901 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3810 3902 mutex_exit(&service_queue);
3811 3903 if (removed) {
3812 3904 sq->sq_svcflags &= ~SQ_BGTHREAD;
3813 3905 sq->sq_servcount = 0;
3814 3906 STRSTAT(sqremoved);
3815 3907 goto done;
3816 3908 }
3817 3909 }
3818 3910 while (sq->sq_servcount != 0) {
3819 3911 sq->sq_flags |= SQ_WANTWAKEUP;
3820 3912 cv_wait(&sq->sq_wait, SQLOCK(sq));
3821 3913 }
3822 3914 done:
3823 3915 mutex_exit(SQLOCK(sq));
3824 3916 }
3825 3917
3826 3918 /*
3827 3919 * Put a syncq on the list of syncq's to be serviced by the sqthread.
3828 3920 * Add the argument to the end of the sqhead list and set the flag
3829 3921 * indicating this syncq has been enabled. If it has already been
3830 3922 * enabled, don't do anything.
3831 3923 * This routine assumes that SQLOCK is held.
3832 3924 * NOTE that the lock order is to have the SQLOCK first,
3833 3925 * so if the service_syncq lock is held, we need to release it
3834 3926 * before acquiring the SQLOCK (mostly relevant for the background
3835 3927 * thread, and this seems to be common among the STREAMS global locks).
3836 3928 * Note that the sq_svcflags are protected by the SQLOCK.
3837 3929 */
3838 3930 void
3839 3931 sqenable(syncq_t *sq)
3840 3932 {
3841 3933 /*
3842 3934 * This is probably not important except for where I believe it
3843 3935 * is being called. At that point, it should be held (and it
3844 3936 * is a pain to release it just for this routine, so don't do
3845 3937 * it).
3846 3938 */
3847 3939 ASSERT(MUTEX_HELD(SQLOCK(sq)));
3848 3940
3849 3941 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3850 3942 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3851 3943
3852 3944 /*
3853 3945 * Do not put on list if background thread is scheduled or
3854 3946 * syncq is disabled.
3855 3947 */
3856 3948 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3857 3949 return;
3858 3950
3859 3951 /*
3860 3952 * Check whether we should enable sq at all.
3861 3953 * Non PERMOD syncqs may be drained by at most one thread.
3862 3954 * PERMOD syncqs may be drained by several threads but we limit the
3863 3955 * total amount to the lesser of
3864 3956 * Number of queues on the squeue and
3865 3957 * Number of CPUs.
3866 3958 */
3867 3959 if (sq->sq_servcount != 0) {
3868 3960 if (((sq->sq_type & SQ_PERMOD) == 0) ||
3869 3961 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3870 3962 STRSTAT(sqtoomany);
3871 3963 return;
3872 3964 }
3873 3965 }
3874 3966
3875 3967 sq->sq_tstamp = ddi_get_lbolt();
3876 3968 STRSTAT(sqenables);
3877 3969
3878 3970 /* Attempt a taskq dispatch */
3879 3971 sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3880 3972 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3881 3973 if (sq->sq_servid != NULL) {
3882 3974 sq->sq_servcount++;
3883 3975 return;
3884 3976 }
3885 3977
3886 3978 /*
3887 3979 * This taskq dispatch failed, but a previous one may have succeeded.
3888 3980 * Don't try to schedule on the background thread whilst there is
3889 3981 * outstanding taskq processing.
3890 3982 */
3891 3983 if (sq->sq_servcount != 0)
3892 3984 return;
3893 3985
3894 3986 /*
3895 3987 * System is low on resources and can't perform a non-sleeping
3896 3988 * dispatch. Schedule the syncq for a background thread and mark the
3897 3989 * syncq to avoid any further taskq dispatch attempts.
3898 3990 */
3899 3991 mutex_enter(&service_queue);
3900 3992 STRSTAT(taskqfails);
3901 3993 ENQUEUE(sq, sqhead, sqtail, sq_next);
3902 3994 sq->sq_svcflags |= SQ_BGTHREAD;
3903 3995 sq->sq_servcount = 1;
3904 3996 cv_signal(&syncqs_to_run);
3905 3997 mutex_exit(&service_queue);
3906 3998 }
3907 3999
3908 4000 /*
3909 4001 * Note: fifo_close() depends on the mblk_t on the queue being freed
3910 4002 * asynchronously. The asynchronous freeing of messages breaks the
3911 4003 * recursive call chain of fifo_close() while there are I_SENDFD type of
3912 4004 * messages referring to other file pointers on the queue. Then when
3913 4005 * closing pipes it can avoid stack overflow in case of daisy-chained
3914 4006 * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3915 4007 * share the same fifolock_t).
3916 4008 *
3917 4009 * No need to kpreempt_disable to access cpu_seqid. If we migrate and
3918 4010 * the esb queue does not match the new CPU, that is OK.
3919 4011 */
3920 4012 void
3921 4013 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3922 4014 {
3923 4015 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3924 4016 esb_queue_t *eqp;
3925 4017
3926 4018 ASSERT(dbp->db_mblk == mp);
3927 4019 ASSERT(qindex < esbq_nelem);
3928 4020
3929 4021 eqp = system_esbq_array;
3930 4022 if (eqp != NULL) {
3931 4023 eqp += qindex;
3932 4024 } else {
3933 4025 mutex_enter(&esbq_lock);
3934 4026 if (kmem_ready && system_esbq_array == NULL)
3935 4027 system_esbq_array = (esb_queue_t *)kmem_zalloc(
3936 4028 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3937 4029 mutex_exit(&esbq_lock);
3938 4030 eqp = system_esbq_array;
3939 4031 if (eqp != NULL)
3940 4032 eqp += qindex;
3941 4033 else
3942 4034 eqp = &system_esbq;
3943 4035 }
3944 4036
3945 4037 /*
3946 4038 * Check data sanity. The dblock should have non-empty free function.
3947 4039 * It is better to panic here then later when the dblock is freed
3948 4040 * asynchronously when the context is lost.
3949 4041 */
3950 4042 if (dbp->db_frtnp->free_func == NULL) {
3951 4043 panic("freebs_enqueue: dblock %p has a NULL free callback",
3952 4044 (void *)dbp);
3953 4045 }
3954 4046
3955 4047 mutex_enter(&eqp->eq_lock);
3956 4048 /* queue the new mblk on the esballoc queue */
3957 4049 if (eqp->eq_head == NULL) {
3958 4050 eqp->eq_head = eqp->eq_tail = mp;
3959 4051 } else {
3960 4052 eqp->eq_tail->b_next = mp;
3961 4053 eqp->eq_tail = mp;
3962 4054 }
3963 4055 eqp->eq_len++;
3964 4056
3965 4057 /* If we're the first thread to reach the threshold, process */
3966 4058 if (eqp->eq_len >= esbq_max_qlen &&
3967 4059 !(eqp->eq_flags & ESBQ_PROCESSING))
3968 4060 esballoc_process_queue(eqp);
3969 4061
3970 4062 esballoc_set_timer(eqp, esbq_timeout);
3971 4063 mutex_exit(&eqp->eq_lock);
3972 4064 }
3973 4065
3974 4066 static void
3975 4067 esballoc_process_queue(esb_queue_t *eqp)
3976 4068 {
3977 4069 mblk_t *mp;
3978 4070
3979 4071 ASSERT(MUTEX_HELD(&eqp->eq_lock));
3980 4072
3981 4073 eqp->eq_flags |= ESBQ_PROCESSING;
3982 4074
3983 4075 do {
3984 4076 /*
3985 4077 * Detach the message chain for processing.
3986 4078 */
3987 4079 mp = eqp->eq_head;
3988 4080 eqp->eq_tail->b_next = NULL;
3989 4081 eqp->eq_head = eqp->eq_tail = NULL;
3990 4082 eqp->eq_len = 0;
3991 4083 mutex_exit(&eqp->eq_lock);
3992 4084
3993 4085 /*
3994 4086 * Process the message chain.
3995 4087 */
3996 4088 esballoc_enqueue_mblk(mp);
3997 4089 mutex_enter(&eqp->eq_lock);
3998 4090 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
3999 4091
4000 4092 eqp->eq_flags &= ~ESBQ_PROCESSING;
4001 4093 }
4002 4094
4003 4095 /*
4004 4096 * taskq callback routine to free esballoced mblk's
4005 4097 */
4006 4098 static void
4007 4099 esballoc_mblk_free(mblk_t *mp)
4008 4100 {
4009 4101 mblk_t *nextmp;
4010 4102
4011 4103 for (; mp != NULL; mp = nextmp) {
4012 4104 nextmp = mp->b_next;
4013 4105 mp->b_next = NULL;
4014 4106 mblk_free(mp);
4015 4107 }
4016 4108 }
4017 4109
4018 4110 static void
4019 4111 esballoc_enqueue_mblk(mblk_t *mp)
4020 4112 {
4021 4113
4022 4114 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4023 4115 TQ_NOSLEEP) == NULL) {
4024 4116 mblk_t *first_mp = mp;
4025 4117 /*
4026 4118 * System is low on resources and can't perform a non-sleeping
4027 4119 * dispatch. Schedule for a background thread.
4028 4120 */
4029 4121 mutex_enter(&service_queue);
4030 4122 STRSTAT(taskqfails);
4031 4123
4032 4124 while (mp->b_next != NULL)
4033 4125 mp = mp->b_next;
4034 4126
4035 4127 mp->b_next = freebs_list;
4036 4128 freebs_list = first_mp;
4037 4129 cv_signal(&services_to_run);
4038 4130 mutex_exit(&service_queue);
4039 4131 }
4040 4132 }
4041 4133
4042 4134 static void
4043 4135 esballoc_timer(void *arg)
4044 4136 {
4045 4137 esb_queue_t *eqp = arg;
4046 4138
4047 4139 mutex_enter(&eqp->eq_lock);
4048 4140 eqp->eq_flags &= ~ESBQ_TIMER;
4049 4141
4050 4142 if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4051 4143 eqp->eq_len > 0)
4052 4144 esballoc_process_queue(eqp);
4053 4145
4054 4146 esballoc_set_timer(eqp, esbq_timeout);
4055 4147 mutex_exit(&eqp->eq_lock);
4056 4148 }
4057 4149
4058 4150 static void
4059 4151 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4060 4152 {
4061 4153 ASSERT(MUTEX_HELD(&eqp->eq_lock));
4062 4154
4063 4155 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4064 4156 (void) timeout(esballoc_timer, eqp, eq_timeout);
4065 4157 eqp->eq_flags |= ESBQ_TIMER;
4066 4158 }
4067 4159 }
4068 4160
4069 4161 /*
4070 4162 * Setup esbq array length based upon NCPU scaled by CPUs per
4071 4163 * queue. Use static system_esbq until kmem_ready and we can
4072 4164 * create an array in freebs_enqueue().
4073 4165 */
4074 4166 void
4075 4167 esballoc_queue_init(void)
4076 4168 {
4077 4169 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4078 4170 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4079 4171 esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4080 4172 system_esbq.eq_len = 0;
4081 4173 system_esbq.eq_head = system_esbq.eq_tail = NULL;
4082 4174 system_esbq.eq_flags = 0;
4083 4175 }
4084 4176
4085 4177 /*
4086 4178 * Set the QBACK or QB_BACK flag in the given queue for
4087 4179 * the given priority band.
4088 4180 */
4089 4181 void
4090 4182 setqback(queue_t *q, unsigned char pri)
4091 4183 {
4092 4184 int i;
4093 4185 qband_t *qbp;
4094 4186 qband_t **qbpp;
4095 4187
4096 4188 ASSERT(MUTEX_HELD(QLOCK(q)));
4097 4189 if (pri != 0) {
4098 4190 if (pri > q->q_nband) {
4099 4191 qbpp = &q->q_bandp;
4100 4192 while (*qbpp)
4101 4193 qbpp = &(*qbpp)->qb_next;
4102 4194 while (pri > q->q_nband) {
4103 4195 if ((*qbpp = allocband()) == NULL) {
4104 4196 cmn_err(CE_WARN,
4105 4197 "setqback: can't allocate qband\n");
4106 4198 return;
4107 4199 }
4108 4200 (*qbpp)->qb_hiwat = q->q_hiwat;
4109 4201 (*qbpp)->qb_lowat = q->q_lowat;
4110 4202 q->q_nband++;
4111 4203 qbpp = &(*qbpp)->qb_next;
4112 4204 }
4113 4205 }
4114 4206 qbp = q->q_bandp;
4115 4207 i = pri;
4116 4208 while (--i)
4117 4209 qbp = qbp->qb_next;
4118 4210 qbp->qb_flag |= QB_BACK;
4119 4211 } else {
4120 4212 q->q_flag |= QBACK;
4121 4213 }
4122 4214 }
4123 4215
4124 4216 int
4125 4217 strcopyin(void *from, void *to, size_t len, int copyflag)
4126 4218 {
4127 4219 if (copyflag & U_TO_K) {
4128 4220 ASSERT((copyflag & K_TO_K) == 0);
4129 4221 if (copyin(from, to, len))
4130 4222 return (EFAULT);
4131 4223 } else {
4132 4224 ASSERT(copyflag & K_TO_K);
4133 4225 bcopy(from, to, len);
4134 4226 }
4135 4227 return (0);
4136 4228 }
4137 4229
4138 4230 int
4139 4231 strcopyout(void *from, void *to, size_t len, int copyflag)
4140 4232 {
4141 4233 if (copyflag & U_TO_K) {
4142 4234 if (copyout(from, to, len))
4143 4235 return (EFAULT);
4144 4236 } else {
4145 4237 ASSERT(copyflag & K_TO_K);
4146 4238 bcopy(from, to, len);
4147 4239 }
4148 4240 return (0);
4149 4241 }
4150 4242
4151 4243 /*
4152 4244 * strsignal_nolock() posts a signal to the process(es) at the stream head.
4153 4245 * It assumes that the stream head lock is already held, whereas strsignal()
4154 4246 * acquires the lock first. This routine was created because a few callers
4155 4247 * release the stream head lock before calling only to re-acquire it after
4156 4248 * it returns.
4157 4249 */
4158 4250 void
4159 4251 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4160 4252 {
4161 4253 ASSERT(MUTEX_HELD(&stp->sd_lock));
4162 4254 switch (sig) {
4163 4255 case SIGPOLL:
4164 4256 if (stp->sd_sigflags & S_MSG)
4165 4257 strsendsig(stp->sd_siglist, S_MSG, band, 0);
4166 4258 break;
4167 4259 default:
4168 4260 if (stp->sd_pgidp)
4169 4261 pgsignal(stp->sd_pgidp, sig);
4170 4262 break;
4171 4263 }
4172 4264 }
4173 4265
4174 4266 void
4175 4267 strsignal(stdata_t *stp, int sig, int32_t band)
4176 4268 {
4177 4269 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4178 4270 "strsignal:%p, %X, %X", stp, sig, band);
4179 4271
4180 4272 mutex_enter(&stp->sd_lock);
4181 4273 switch (sig) {
4182 4274 case SIGPOLL:
4183 4275 if (stp->sd_sigflags & S_MSG)
4184 4276 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4185 4277 break;
4186 4278
4187 4279 default:
4188 4280 if (stp->sd_pgidp) {
4189 4281 pgsignal(stp->sd_pgidp, sig);
4190 4282 }
4191 4283 break;
4192 4284 }
4193 4285 mutex_exit(&stp->sd_lock);
4194 4286 }
4195 4287
4196 4288 void
4197 4289 strhup(stdata_t *stp)
4198 4290 {
4199 4291 ASSERT(mutex_owned(&stp->sd_lock));
4200 4292 pollwakeup(&stp->sd_pollist, POLLHUP);
4201 4293 if (stp->sd_sigflags & S_HANGUP)
4202 4294 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4203 4295 }
4204 4296
4205 4297 /*
4206 4298 * Backenable the first queue upstream from `q' with a service procedure.
4207 4299 */
4208 4300 void
4209 4301 backenable(queue_t *q, uchar_t pri)
4210 4302 {
4211 4303 queue_t *nq;
4212 4304
4213 4305 /*
4214 4306 * Our presence might not prevent other modules in our own
4215 4307 * stream from popping/pushing since the caller of getq might not
4216 4308 * have a claim on the queue (some drivers do a getq on somebody
4217 4309 * else's queue - they know that the queue itself is not going away
4218 4310 * but the framework has to guarantee q_next in that stream).
4219 4311 */
4220 4312 claimstr(q);
4221 4313
4222 4314 /* Find nearest back queue with service proc */
4223 4315 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4224 4316 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4225 4317 }
4226 4318
4227 4319 if (nq) {
4228 4320 kthread_t *freezer;
4229 4321 /*
4230 4322 * backenable can be called either with no locks held
4231 4323 * or with the stream frozen (the latter occurs when a module
4232 4324 * calls rmvq with the stream frozen). If the stream is frozen
4233 4325 * by the caller the caller will hold all qlocks in the stream.
4234 4326 * Note that a frozen stream doesn't freeze a mated stream,
4235 4327 * so we explicitly check for that.
4236 4328 */
4237 4329 freezer = STREAM(q)->sd_freezer;
4238 4330 if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4239 4331 mutex_enter(QLOCK(nq));
4240 4332 }
4241 4333 #ifdef DEBUG
4242 4334 else {
4243 4335 ASSERT(frozenstr(q));
4244 4336 ASSERT(MUTEX_HELD(QLOCK(q)));
4245 4337 ASSERT(MUTEX_HELD(QLOCK(nq)));
4246 4338 }
4247 4339 #endif
4248 4340 setqback(nq, pri);
4249 4341 qenable_locked(nq);
4250 4342 if (freezer != curthread || STREAM(q) != STREAM(nq))
4251 4343 mutex_exit(QLOCK(nq));
4252 4344 }
4253 4345 releasestr(q);
4254 4346 }
4255 4347
4256 4348 /*
4257 4349 * Return the appropriate errno when one of flags_to_check is set
4258 4350 * in sd_flags. Uses the exported error routines if they are set.
4259 4351 * Will return 0 if non error is set (or if the exported error routines
4260 4352 * do not return an error).
4261 4353 *
4262 4354 * If there is both a read and write error to check, we prefer the read error.
4263 4355 * Also, give preference to recorded errno's over the error functions.
4264 4356 * The flags that are handled are:
4265 4357 * STPLEX return EINVAL
4266 4358 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST)
4267 4359 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST)
4268 4360 * STRHUP return sd_werror
4269 4361 *
4270 4362 * If the caller indicates that the operation is a peek, a nonpersistent error
4271 4363 * is not cleared.
4272 4364 */
4273 4365 int
4274 4366 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4275 4367 {
4276 4368 int32_t sd_flag = stp->sd_flag & flags_to_check;
4277 4369 int error = 0;
4278 4370
4279 4371 ASSERT(MUTEX_HELD(&stp->sd_lock));
4280 4372 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4281 4373 if (sd_flag & STPLEX)
4282 4374 error = EINVAL;
4283 4375 else if (sd_flag & STRDERR) {
4284 4376 error = stp->sd_rerror;
4285 4377 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4286 4378 /*
4287 4379 * Read errors are non-persistent i.e. discarded once
4288 4380 * returned to a non-peeking caller,
4289 4381 */
4290 4382 stp->sd_rerror = 0;
4291 4383 stp->sd_flag &= ~STRDERR;
4292 4384 }
4293 4385 if (error == 0 && stp->sd_rderrfunc != NULL) {
4294 4386 int clearerr = 0;
4295 4387
4296 4388 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4297 4389 &clearerr);
4298 4390 if (clearerr) {
4299 4391 stp->sd_flag &= ~STRDERR;
4300 4392 stp->sd_rderrfunc = NULL;
4301 4393 }
4302 4394 }
4303 4395 } else if (sd_flag & STWRERR) {
4304 4396 error = stp->sd_werror;
4305 4397 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4306 4398 /*
4307 4399 * Write errors are non-persistent i.e. discarded once
4308 4400 * returned to a non-peeking caller,
4309 4401 */
4310 4402 stp->sd_werror = 0;
4311 4403 stp->sd_flag &= ~STWRERR;
4312 4404 }
4313 4405 if (error == 0 && stp->sd_wrerrfunc != NULL) {
4314 4406 int clearerr = 0;
4315 4407
4316 4408 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4317 4409 &clearerr);
4318 4410 if (clearerr) {
4319 4411 stp->sd_flag &= ~STWRERR;
4320 4412 stp->sd_wrerrfunc = NULL;
4321 4413 }
4322 4414 }
4323 4415 } else if (sd_flag & STRHUP) {
4324 4416 /* sd_werror set when STRHUP */
4325 4417 error = stp->sd_werror;
4326 4418 }
4327 4419 return (error);
4328 4420 }
4329 4421
4330 4422
4331 4423 /*
4332 4424 * Single-thread open/close/push/pop
4333 4425 * for twisted streams also
4334 4426 */
4335 4427 int
4336 4428 strstartplumb(stdata_t *stp, int flag, int cmd)
4337 4429 {
4338 4430 int waited = 1;
4339 4431 int error = 0;
4340 4432
4341 4433 if (STRMATED(stp)) {
4342 4434 struct stdata *stmatep = stp->sd_mate;
4343 4435
4344 4436 STRLOCKMATES(stp);
4345 4437 while (waited) {
4346 4438 waited = 0;
4347 4439 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4348 4440 if ((cmd == I_POP) &&
4349 4441 (flag & (FNDELAY|FNONBLOCK))) {
4350 4442 STRUNLOCKMATES(stp);
4351 4443 return (EAGAIN);
4352 4444 }
4353 4445 waited = 1;
4354 4446 mutex_exit(&stp->sd_lock);
4355 4447 if (!cv_wait_sig(&stmatep->sd_monitor,
4356 4448 &stmatep->sd_lock)) {
4357 4449 mutex_exit(&stmatep->sd_lock);
4358 4450 return (EINTR);
4359 4451 }
4360 4452 mutex_exit(&stmatep->sd_lock);
4361 4453 STRLOCKMATES(stp);
4362 4454 }
4363 4455 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4364 4456 if ((cmd == I_POP) &&
4365 4457 (flag & (FNDELAY|FNONBLOCK))) {
4366 4458 STRUNLOCKMATES(stp);
4367 4459 return (EAGAIN);
4368 4460 }
4369 4461 waited = 1;
4370 4462 mutex_exit(&stmatep->sd_lock);
4371 4463 if (!cv_wait_sig(&stp->sd_monitor,
4372 4464 &stp->sd_lock)) {
4373 4465 mutex_exit(&stp->sd_lock);
4374 4466 return (EINTR);
4375 4467 }
4376 4468 mutex_exit(&stp->sd_lock);
4377 4469 STRLOCKMATES(stp);
4378 4470 }
4379 4471 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4380 4472 error = strgeterr(stp,
4381 4473 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4382 4474 if (error != 0) {
4383 4475 STRUNLOCKMATES(stp);
4384 4476 return (error);
4385 4477 }
4386 4478 }
4387 4479 }
4388 4480 stp->sd_flag |= STRPLUMB;
4389 4481 STRUNLOCKMATES(stp);
4390 4482 } else {
4391 4483 mutex_enter(&stp->sd_lock);
4392 4484 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4393 4485 if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4394 4486 (flag & (FNDELAY|FNONBLOCK))) {
4395 4487 mutex_exit(&stp->sd_lock);
4396 4488 return (EAGAIN);
4397 4489 }
4398 4490 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4399 4491 mutex_exit(&stp->sd_lock);
4400 4492 return (EINTR);
4401 4493 }
4402 4494 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4403 4495 error = strgeterr(stp,
4404 4496 STRDERR|STWRERR|STRHUP|STPLEX, 0);
4405 4497 if (error != 0) {
4406 4498 mutex_exit(&stp->sd_lock);
4407 4499 return (error);
4408 4500 }
4409 4501 }
4410 4502 }
4411 4503 stp->sd_flag |= STRPLUMB;
4412 4504 mutex_exit(&stp->sd_lock);
4413 4505 }
4414 4506 return (0);
4415 4507 }
4416 4508
4417 4509 /*
4418 4510 * Complete the plumbing operation associated with stream `stp'.
4419 4511 */
4420 4512 void
4421 4513 strendplumb(stdata_t *stp)
4422 4514 {
4423 4515 ASSERT(MUTEX_HELD(&stp->sd_lock));
4424 4516 ASSERT(stp->sd_flag & STRPLUMB);
4425 4517 stp->sd_flag &= ~STRPLUMB;
4426 4518 cv_broadcast(&stp->sd_monitor);
4427 4519 }
4428 4520
4429 4521 /*
4430 4522 * This describes how the STREAMS framework handles synchronization
4431 4523 * during open/push and close/pop.
4432 4524 * The key interfaces for open and close are qprocson and qprocsoff,
4433 4525 * respectively. While the close case in general is harder both open
4434 4526 * have close have significant similarities.
4435 4527 *
4436 4528 * During close the STREAMS framework has to both ensure that there
4437 4529 * are no stale references to the queue pair (and syncq) that
4438 4530 * are being closed and also provide the guarantees that are documented
4439 4531 * in qprocsoff(9F).
4440 4532 * If there are stale references to the queue that is closing it can
4441 4533 * result in kernel memory corruption or kernel panics.
4442 4534 *
4443 4535 * Note that is it up to the module/driver to ensure that it itself
4444 4536 * does not have any stale references to the closing queues once its close
4445 4537 * routine returns. This includes:
4446 4538 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4447 4539 * associated with the queues. For timeout and bufcall callbacks the
4448 4540 * module/driver also has to ensure (or wait for) any callbacks that
4449 4541 * are in progress.
4450 4542 * - If the module/driver is using esballoc it has to ensure that any
4451 4543 * esballoc free functions do not refer to a queue that has closed.
4452 4544 * (Note that in general the close routine can not wait for the esballoc'ed
4453 4545 * messages to be freed since that can cause a deadlock.)
4454 4546 * - Cancelling any interrupts that refer to the closing queues and
4455 4547 * also ensuring that there are no interrupts in progress that will
4456 4548 * refer to the closing queues once the close routine returns.
4457 4549 * - For multiplexors removing any driver global state that refers to
4458 4550 * the closing queue and also ensuring that there are no threads in
4459 4551 * the multiplexor that has picked up a queue pointer but not yet
4460 4552 * finished using it.
4461 4553 *
4462 4554 * In addition, a driver/module can only reference the q_next pointer
4463 4555 * in its open, close, put, or service procedures or in a
4464 4556 * qtimeout/qbufcall callback procedure executing "on" the correct
4465 4557 * stream. Thus it can not reference the q_next pointer in an interrupt
4466 4558 * routine or a timeout, bufcall or esballoc callback routine. Likewise
4467 4559 * it can not reference q_next of a different queue e.g. in a mux that
4468 4560 * passes messages from one queues put/service procedure to another queue.
4469 4561 * In all the cases when the driver/module can not access the q_next
4470 4562 * field it must use the *next* versions e.g. canputnext instead of
4471 4563 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4472 4564 *
4473 4565 *
4474 4566 * Assuming that the driver/module conforms to the above constraints
4475 4567 * the STREAMS framework has to avoid stale references to q_next for all
4476 4568 * the framework internal cases which include (but are not limited to):
4477 4569 * - Threads in canput/canputnext/backenable and elsewhere that are
4478 4570 * walking q_next.
4479 4571 * - Messages on a syncq that have a reference to the queue through b_queue.
4480 4572 * - Messages on an outer perimeter (syncq) that have a reference to the
4481 4573 * queue through b_queue.
4482 4574 * - Threads that use q_nfsrv (e.g. canput) to find a queue.
4483 4575 * Note that only canput and bcanput use q_nfsrv without any locking.
4484 4576 *
4485 4577 * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4486 4578 * after qprocsoff returns, the framework has to ensure that no threads can
4487 4579 * enter the put or service routines for the closing read or write-side queue.
4488 4580 * In addition to preventing "direct" entry into the put procedures
4489 4581 * the framework also has to prevent messages being drained from
4490 4582 * the syncq or the outer perimeter.
4491 4583 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4492 4584 * mechanism to prevent qwriter(PERIM_OUTER) from running after
4493 4585 * qprocsoff has returned.
4494 4586 * Note that if a module/driver uses put(9F) on one of its own queues
4495 4587 * it is up to the module/driver to ensure that the put() doesn't
4496 4588 * get called when the queue is closing.
4497 4589 *
4498 4590 *
4499 4591 * The framework aspects of the above "contract" is implemented by
4500 4592 * qprocsoff, removeq, and strlock:
4501 4593 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4502 4594 * entering the service procedures.
4503 4595 * - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4504 4596 * canputnext, backenable etc from dereferencing the q_next that will
4505 4597 * soon change.
4506 4598 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4507 4599 * or other q_next walker that uses claimstr/releasestr to finish.
4508 4600 * - optionally for every syncq in the stream strlock acquires all the
4509 4601 * sq_lock's and waits for all sq_counts to drop to a value that indicates
4510 4602 * that no thread executes in the put or service procedures and that no
4511 4603 * thread is draining into the module/driver. This ensures that no
4512 4604 * open, close, put, service, or qtimeout/qbufcall callback procedure is
4513 4605 * currently executing hence no such thread can end up with the old stale
4514 4606 * q_next value and no canput/backenable can have the old stale
4515 4607 * q_nfsrv/q_next.
4516 4608 * - qdetach (wait_svc) makes sure that any scheduled or running threads
4517 4609 * have either finished or observed the QWCLOSE flag and gone away.
4518 4610 */
4519 4611
4520 4612
4521 4613 /*
4522 4614 * Get all the locks necessary to change q_next.
4523 4615 *
4524 4616 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4525 4617 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4526 4618 * the only threads inside the syncq are threads currently calling removeq().
4527 4619 * Since threads calling removeq() are in the process of removing their queues
4528 4620 * from the stream, we do not need to worry about them accessing a stale q_next
4529 4621 * pointer and thus we do not need to wait for them to exit (in fact, waiting
4530 4622 * for them can cause deadlock).
4531 4623 *
4532 4624 * This routine is subject to starvation since it does not set any flag to
4533 4625 * prevent threads from entering a module in the stream (i.e. sq_count can
4534 4626 * increase on some syncq while it is waiting on some other syncq).
4535 4627 *
4536 4628 * Assumes that only one thread attempts to call strlock for a given
4537 4629 * stream. If this is not the case the two threads would deadlock.
4538 4630 * This assumption is guaranteed since strlock is only called by insertq
4539 4631 * and removeq and streams plumbing changes are single-threaded for
4540 4632 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4541 4633 *
4542 4634 * For pipes, it is not difficult to atomically designate a pair of streams
4543 4635 * to be mated. Once mated atomically by the framework the twisted pair remain
4544 4636 * configured that way until dismantled atomically by the framework.
4545 4637 * When plumbing takes place on a twisted stream it is necessary to ensure that
4546 4638 * this operation is done exclusively on the twisted stream since two such
4547 4639 * operations, each initiated on different ends of the pipe will deadlock
4548 4640 * waiting for each other to complete.
4549 4641 *
4550 4642 * On entry, no locks should be held.
4551 4643 * The locks acquired and held by strlock depends on a few factors.
4552 4644 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4553 4645 * and held on exit and all sq_count are at an acceptable level.
4554 4646 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4555 4647 * sd_refcnt being zero.
4556 4648 */
4557 4649
4558 4650 static void
4559 4651 strlock(struct stdata *stp, sqlist_t *sqlist)
4560 4652 {
4561 4653 syncql_t *sql, *sql2;
4562 4654 retry:
4563 4655 /*
4564 4656 * Wait for any claimstr to go away.
4565 4657 */
4566 4658 if (STRMATED(stp)) {
4567 4659 struct stdata *stp1, *stp2;
4568 4660
4569 4661 STRLOCKMATES(stp);
4570 4662 /*
4571 4663 * Note that the selection of locking order is not
4572 4664 * important, just that they are always acquired in
4573 4665 * the same order. To assure this, we choose this
4574 4666 * order based on the value of the pointer, and since
4575 4667 * the pointer will not change for the life of this
4576 4668 * pair, we will always grab the locks in the same
4577 4669 * order (and hence, prevent deadlocks).
4578 4670 */
4579 4671 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4580 4672 stp1 = stp;
4581 4673 stp2 = stp->sd_mate;
4582 4674 } else {
4583 4675 stp2 = stp;
4584 4676 stp1 = stp->sd_mate;
4585 4677 }
4586 4678 mutex_enter(&stp1->sd_reflock);
4587 4679 if (stp1->sd_refcnt > 0) {
4588 4680 STRUNLOCKMATES(stp);
4589 4681 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4590 4682 mutex_exit(&stp1->sd_reflock);
4591 4683 goto retry;
4592 4684 }
4593 4685 mutex_enter(&stp2->sd_reflock);
4594 4686 if (stp2->sd_refcnt > 0) {
4595 4687 STRUNLOCKMATES(stp);
4596 4688 mutex_exit(&stp1->sd_reflock);
4597 4689 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4598 4690 mutex_exit(&stp2->sd_reflock);
4599 4691 goto retry;
4600 4692 }
4601 4693 STREAM_PUTLOCKS_ENTER(stp1);
4602 4694 STREAM_PUTLOCKS_ENTER(stp2);
4603 4695 } else {
4604 4696 mutex_enter(&stp->sd_lock);
4605 4697 mutex_enter(&stp->sd_reflock);
4606 4698 while (stp->sd_refcnt > 0) {
4607 4699 mutex_exit(&stp->sd_lock);
4608 4700 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4609 4701 if (mutex_tryenter(&stp->sd_lock) == 0) {
4610 4702 mutex_exit(&stp->sd_reflock);
4611 4703 mutex_enter(&stp->sd_lock);
4612 4704 mutex_enter(&stp->sd_reflock);
4613 4705 }
4614 4706 }
4615 4707 STREAM_PUTLOCKS_ENTER(stp);
4616 4708 }
4617 4709
4618 4710 if (sqlist == NULL)
4619 4711 return;
4620 4712
4621 4713 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4622 4714 syncq_t *sq = sql->sql_sq;
4623 4715 uint16_t count;
4624 4716
4625 4717 mutex_enter(SQLOCK(sq));
4626 4718 count = sq->sq_count;
4627 4719 ASSERT(sq->sq_rmqcount <= count);
4628 4720 SQ_PUTLOCKS_ENTER(sq);
4629 4721 SUM_SQ_PUTCOUNTS(sq, count);
4630 4722 if (count == sq->sq_rmqcount)
4631 4723 continue;
4632 4724
4633 4725 /* Failed - drop all locks that we have acquired so far */
4634 4726 if (STRMATED(stp)) {
4635 4727 STREAM_PUTLOCKS_EXIT(stp);
4636 4728 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4637 4729 STRUNLOCKMATES(stp);
4638 4730 mutex_exit(&stp->sd_reflock);
4639 4731 mutex_exit(&stp->sd_mate->sd_reflock);
4640 4732 } else {
4641 4733 STREAM_PUTLOCKS_EXIT(stp);
4642 4734 mutex_exit(&stp->sd_lock);
4643 4735 mutex_exit(&stp->sd_reflock);
4644 4736 }
4645 4737 for (sql2 = sqlist->sqlist_head; sql2 != sql;
4646 4738 sql2 = sql2->sql_next) {
4647 4739 SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4648 4740 mutex_exit(SQLOCK(sql2->sql_sq));
4649 4741 }
4650 4742
4651 4743 /*
4652 4744 * The wait loop below may starve when there are many threads
4653 4745 * claiming the syncq. This is especially a problem with permod
4654 4746 * syncqs (IP). To lessen the impact of the problem we increment
4655 4747 * sq_needexcl and clear fastbits so that putnexts will slow
4656 4748 * down and call sqenable instead of draining right away.
4657 4749 */
4658 4750 sq->sq_needexcl++;
4659 4751 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4660 4752 while (count > sq->sq_rmqcount) {
4661 4753 sq->sq_flags |= SQ_WANTWAKEUP;
4662 4754 SQ_PUTLOCKS_EXIT(sq);
4663 4755 cv_wait(&sq->sq_wait, SQLOCK(sq));
4664 4756 count = sq->sq_count;
4665 4757 SQ_PUTLOCKS_ENTER(sq);
4666 4758 SUM_SQ_PUTCOUNTS(sq, count);
4667 4759 }
4668 4760 sq->sq_needexcl--;
4669 4761 if (sq->sq_needexcl == 0)
4670 4762 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4671 4763 SQ_PUTLOCKS_EXIT(sq);
4672 4764 ASSERT(count == sq->sq_rmqcount);
4673 4765 mutex_exit(SQLOCK(sq));
4674 4766 goto retry;
4675 4767 }
4676 4768 }
4677 4769
4678 4770 /*
4679 4771 * Drop all the locks that strlock acquired.
4680 4772 */
4681 4773 static void
4682 4774 strunlock(struct stdata *stp, sqlist_t *sqlist)
4683 4775 {
4684 4776 syncql_t *sql;
4685 4777
4686 4778 if (STRMATED(stp)) {
4687 4779 STREAM_PUTLOCKS_EXIT(stp);
4688 4780 STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4689 4781 STRUNLOCKMATES(stp);
4690 4782 mutex_exit(&stp->sd_reflock);
4691 4783 mutex_exit(&stp->sd_mate->sd_reflock);
4692 4784 } else {
4693 4785 STREAM_PUTLOCKS_EXIT(stp);
4694 4786 mutex_exit(&stp->sd_lock);
4695 4787 mutex_exit(&stp->sd_reflock);
4696 4788 }
4697 4789
4698 4790 if (sqlist == NULL)
4699 4791 return;
4700 4792
4701 4793 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4702 4794 SQ_PUTLOCKS_EXIT(sql->sql_sq);
4703 4795 mutex_exit(SQLOCK(sql->sql_sq));
4704 4796 }
4705 4797 }
4706 4798
4707 4799 /*
4708 4800 * When the module has service procedure, we need check if the next
4709 4801 * module which has service procedure is in flow control to trigger
4710 4802 * the backenable.
4711 4803 */
4712 4804 static void
4713 4805 backenable_insertedq(queue_t *q)
4714 4806 {
4715 4807 qband_t *qbp;
4716 4808
4717 4809 claimstr(q);
4718 4810 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4719 4811 if (q->q_next->q_nfsrv->q_flag & QWANTW)
4720 4812 backenable(q, 0);
4721 4813
4722 4814 qbp = q->q_next->q_nfsrv->q_bandp;
4723 4815 for (; qbp != NULL; qbp = qbp->qb_next)
4724 4816 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4725 4817 backenable(q, qbp->qb_first->b_band);
4726 4818 }
4727 4819 releasestr(q);
4728 4820 }
4729 4821
4730 4822 /*
4731 4823 * Given two read queues, insert a new single one after another.
4732 4824 *
4733 4825 * This routine acquires all the necessary locks in order to change
4734 4826 * q_next and related pointer using strlock().
4735 4827 * It depends on the stream head ensuring that there are no concurrent
4736 4828 * insertq or removeq on the same stream. The stream head ensures this
4737 4829 * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4738 4830 *
4739 4831 * Note that no syncq locks are held during the q_next change. This is
4740 4832 * applied to all streams since, unlike removeq, there is no problem of stale
4741 4833 * pointers when adding a module to the stream. Thus drivers/modules that do a
4742 4834 * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4743 4835 * applied this optimization to all streams.
4744 4836 */
4745 4837 void
4746 4838 insertq(struct stdata *stp, queue_t *new)
4747 4839 {
4748 4840 queue_t *after;
4749 4841 queue_t *wafter;
4750 4842 queue_t *wnew = _WR(new);
4751 4843 boolean_t have_fifo = B_FALSE;
4752 4844
4753 4845 if (new->q_flag & _QINSERTING) {
4754 4846 ASSERT(stp->sd_vnode->v_type != VFIFO);
4755 4847 after = new->q_next;
4756 4848 wafter = _WR(new->q_next);
4757 4849 } else {
4758 4850 after = _RD(stp->sd_wrq);
4759 4851 wafter = stp->sd_wrq;
4760 4852 }
4761 4853
4762 4854 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4763 4855 "insertq:%p, %p", after, new);
4764 4856 ASSERT(after->q_flag & QREADR);
4765 4857 ASSERT(new->q_flag & QREADR);
4766 4858
4767 4859 strlock(stp, NULL);
4768 4860
4769 4861 /* Do we have a FIFO? */
4770 4862 if (wafter->q_next == after) {
4771 4863 have_fifo = B_TRUE;
4772 4864 wnew->q_next = new;
4773 4865 } else {
4774 4866 wnew->q_next = wafter->q_next;
4775 4867 }
4776 4868 new->q_next = after;
4777 4869
4778 4870 set_nfsrv_ptr(new, wnew, after, wafter);
4779 4871 /*
4780 4872 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4781 4873 * so only reset this flag after calling it.
4782 4874 */
4783 4875 new->q_flag &= ~_QINSERTING;
4784 4876
4785 4877 if (have_fifo) {
4786 4878 wafter->q_next = wnew;
4787 4879 } else {
4788 4880 if (wafter->q_next)
4789 4881 _OTHERQ(wafter->q_next)->q_next = new;
4790 4882 wafter->q_next = wnew;
4791 4883 }
4792 4884
4793 4885 set_qend(new);
4794 4886 /* The QEND flag might have to be updated for the upstream guy */
4795 4887 set_qend(after);
4796 4888
4797 4889 ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4798 4890 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4799 4891 ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4800 4892 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4801 4893 strsetuio(stp);
4802 4894
4803 4895 /*
4804 4896 * If this was a module insertion, bump the push count.
4805 4897 */
4806 4898 if (!(new->q_flag & QISDRV))
4807 4899 stp->sd_pushcnt++;
4808 4900
4809 4901 strunlock(stp, NULL);
4810 4902
4811 4903 /* check if the write Q needs backenable */
4812 4904 backenable_insertedq(wnew);
4813 4905
4814 4906 /* check if the read Q needs backenable */
4815 4907 backenable_insertedq(new);
4816 4908 }
4817 4909
4818 4910 /*
4819 4911 * Given a read queue, unlink it from any neighbors.
4820 4912 *
4821 4913 * This routine acquires all the necessary locks in order to
4822 4914 * change q_next and related pointers and also guard against
4823 4915 * stale references (e.g. through q_next) to the queue that
4824 4916 * is being removed. It also plays part of the role in ensuring
4825 4917 * that the module's/driver's put procedure doesn't get called
4826 4918 * after qprocsoff returns.
4827 4919 *
4828 4920 * Removeq depends on the stream head ensuring that there are
4829 4921 * no concurrent insertq or removeq on the same stream. The
4830 4922 * stream head ensures this using the flags STWOPEN, STRCLOSE and
4831 4923 * STRPLUMB.
4832 4924 *
4833 4925 * The set of locks needed to remove the queue is different in
4834 4926 * different cases:
4835 4927 *
4836 4928 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4837 4929 * waiting for the syncq reference count to drop to 0 indicating that no
4838 4930 * non-close threads are present anywhere in the stream. This ensures that any
4839 4931 * module/driver can reference q_next in its open, close, put, or service
4840 4932 * procedures.
4841 4933 *
4842 4934 * The sq_rmqcount counter tracks the number of threads inside removeq().
4843 4935 * strlock() ensures that there is either no threads executing inside perimeter
4844 4936 * or there is only a thread calling qprocsoff().
4845 4937 *
4846 4938 * strlock() compares the value of sq_count with the number of threads inside
4847 4939 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4848 4940 * any threads waiting in strlock() when the sq_rmqcount increases.
4849 4941 */
4850 4942
4851 4943 void
4852 4944 removeq(queue_t *qp)
4853 4945 {
4854 4946 queue_t *wqp = _WR(qp);
4855 4947 struct stdata *stp = STREAM(qp);
4856 4948 sqlist_t *sqlist = NULL;
4857 4949 boolean_t isdriver;
4858 4950 int moved;
4859 4951 syncq_t *sq = qp->q_syncq;
4860 4952 syncq_t *wsq = wqp->q_syncq;
4861 4953
4862 4954 ASSERT(stp);
4863 4955
4864 4956 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4865 4957 "removeq:%p %p", qp, wqp);
4866 4958 ASSERT(qp->q_flag&QREADR);
4867 4959
4868 4960 /*
4869 4961 * For queues using Synchronous streams, we must wait for all threads in
4870 4962 * rwnext() to drain out before proceeding.
4871 4963 */
4872 4964 if (qp->q_flag & QSYNCSTR) {
4873 4965 /* First, we need wakeup any threads blocked in rwnext() */
4874 4966 mutex_enter(SQLOCK(sq));
4875 4967 if (sq->sq_flags & SQ_WANTWAKEUP) {
4876 4968 sq->sq_flags &= ~SQ_WANTWAKEUP;
4877 4969 cv_broadcast(&sq->sq_wait);
4878 4970 }
4879 4971 mutex_exit(SQLOCK(sq));
4880 4972
4881 4973 if (wsq != sq) {
4882 4974 mutex_enter(SQLOCK(wsq));
4883 4975 if (wsq->sq_flags & SQ_WANTWAKEUP) {
4884 4976 wsq->sq_flags &= ~SQ_WANTWAKEUP;
4885 4977 cv_broadcast(&wsq->sq_wait);
4886 4978 }
4887 4979 mutex_exit(SQLOCK(wsq));
4888 4980 }
4889 4981
4890 4982 mutex_enter(QLOCK(qp));
4891 4983 while (qp->q_rwcnt > 0) {
4892 4984 qp->q_flag |= QWANTRMQSYNC;
4893 4985 cv_wait(&qp->q_wait, QLOCK(qp));
4894 4986 }
4895 4987 mutex_exit(QLOCK(qp));
4896 4988
4897 4989 mutex_enter(QLOCK(wqp));
4898 4990 while (wqp->q_rwcnt > 0) {
4899 4991 wqp->q_flag |= QWANTRMQSYNC;
4900 4992 cv_wait(&wqp->q_wait, QLOCK(wqp));
4901 4993 }
4902 4994 mutex_exit(QLOCK(wqp));
4903 4995 }
4904 4996
4905 4997 mutex_enter(SQLOCK(sq));
4906 4998 sq->sq_rmqcount++;
4907 4999 if (sq->sq_flags & SQ_WANTWAKEUP) {
4908 5000 sq->sq_flags &= ~SQ_WANTWAKEUP;
4909 5001 cv_broadcast(&sq->sq_wait);
4910 5002 }
4911 5003 mutex_exit(SQLOCK(sq));
4912 5004
4913 5005 isdriver = (qp->q_flag & QISDRV);
4914 5006
4915 5007 sqlist = sqlist_build(qp, stp, STRMATED(stp));
4916 5008 strlock(stp, sqlist);
4917 5009
4918 5010 reset_nfsrv_ptr(qp, wqp);
4919 5011
4920 5012 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4921 5013 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4922 5014 /* Do we have a FIFO? */
4923 5015 if (wqp->q_next == qp) {
4924 5016 stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4925 5017 } else {
4926 5018 if (wqp->q_next)
4927 5019 backq(qp)->q_next = qp->q_next;
4928 5020 if (qp->q_next)
4929 5021 backq(wqp)->q_next = wqp->q_next;
4930 5022 }
4931 5023
4932 5024 /* The QEND flag might have to be updated for the upstream guy */
4933 5025 if (qp->q_next)
4934 5026 set_qend(qp->q_next);
4935 5027
4936 5028 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4937 5029 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4938 5030
4939 5031 /*
4940 5032 * Move any messages destined for the put procedures to the next
4941 5033 * syncq in line. Otherwise free them.
4942 5034 */
4943 5035 moved = 0;
4944 5036 /*
4945 5037 * Quick check to see whether there are any messages or events.
4946 5038 */
4947 5039 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4948 5040 moved += propagate_syncq(qp);
4949 5041 if (wqp->q_syncqmsgs != 0 ||
4950 5042 (wqp->q_syncq->sq_flags & SQ_EVENTS))
4951 5043 moved += propagate_syncq(wqp);
4952 5044
4953 5045 strsetuio(stp);
4954 5046
4955 5047 /*
4956 5048 * If this was a module removal, decrement the push count.
4957 5049 */
4958 5050 if (!isdriver)
4959 5051 stp->sd_pushcnt--;
4960 5052
4961 5053 strunlock(stp, sqlist);
4962 5054 sqlist_free(sqlist);
4963 5055
4964 5056 /*
4965 5057 * Make sure any messages that were propagated are drained.
4966 5058 * Also clear any QFULL bit caused by messages that were propagated.
4967 5059 */
4968 5060
4969 5061 if (qp->q_next != NULL) {
4970 5062 clr_qfull(qp);
4971 5063 /*
4972 5064 * For the driver calling qprocsoff, propagate_syncq
4973 5065 * frees all the messages instead of putting it in
4974 5066 * the stream head
4975 5067 */
4976 5068 if (!isdriver && (moved > 0))
4977 5069 emptysq(qp->q_next->q_syncq);
4978 5070 }
4979 5071 if (wqp->q_next != NULL) {
4980 5072 clr_qfull(wqp);
4981 5073 /*
4982 5074 * We come here for any pop of a module except for the
4983 5075 * case of driver being removed. We don't call emptysq
4984 5076 * if we did not move any messages. This will avoid holding
4985 5077 * PERMOD syncq locks in emptysq
4986 5078 */
4987 5079 if (moved > 0)
4988 5080 emptysq(wqp->q_next->q_syncq);
4989 5081 }
4990 5082
4991 5083 mutex_enter(SQLOCK(sq));
4992 5084 sq->sq_rmqcount--;
4993 5085 mutex_exit(SQLOCK(sq));
4994 5086 }
4995 5087
4996 5088 /*
4997 5089 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
4998 5090 * SQ_WRITER) on a syncq.
4999 5091 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5000 5092 * sync queue and waits until sq_count reaches maxcnt.
5001 5093 *
5002 5094 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5003 5095 * does not care about putnext threads that are in the middle of calling put
5004 5096 * entry points.
5005 5097 *
5006 5098 * This routine is used for both inner and outer syncqs.
5007 5099 */
5008 5100 static void
5009 5101 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5010 5102 {
5011 5103 uint16_t count = 0;
5012 5104
5013 5105 mutex_enter(SQLOCK(sq));
5014 5106 /*
5015 5107 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5016 5108 * SQ_FROZEN will be set if there is a frozen stream that has a
5017 5109 * queue which also refers to this "shared" syncq.
5018 5110 * SQ_BLOCKED will be set if there is "off" queue which also
5019 5111 * refers to this "shared" syncq.
5020 5112 */
5021 5113 if (maxcnt != -1) {
5022 5114 count = sq->sq_count;
5023 5115 SQ_PUTLOCKS_ENTER(sq);
5024 5116 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5025 5117 SUM_SQ_PUTCOUNTS(sq, count);
5026 5118 }
5027 5119 sq->sq_needexcl++;
5028 5120 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5029 5121
5030 5122 while ((sq->sq_flags & flag) ||
5031 5123 (maxcnt != -1 && count > (unsigned)maxcnt)) {
5032 5124 sq->sq_flags |= SQ_WANTWAKEUP;
5033 5125 if (maxcnt != -1) {
5034 5126 SQ_PUTLOCKS_EXIT(sq);
5035 5127 }
5036 5128 cv_wait(&sq->sq_wait, SQLOCK(sq));
5037 5129 if (maxcnt != -1) {
5038 5130 count = sq->sq_count;
5039 5131 SQ_PUTLOCKS_ENTER(sq);
5040 5132 SUM_SQ_PUTCOUNTS(sq, count);
5041 5133 }
5042 5134 }
5043 5135 sq->sq_needexcl--;
5044 5136 sq->sq_flags |= flag;
5045 5137 ASSERT(maxcnt == -1 || count == maxcnt);
5046 5138 if (maxcnt != -1) {
5047 5139 if (sq->sq_needexcl == 0) {
5048 5140 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5049 5141 }
5050 5142 SQ_PUTLOCKS_EXIT(sq);
5051 5143 } else if (sq->sq_needexcl == 0) {
5052 5144 SQ_PUTCOUNT_SETFAST(sq);
5053 5145 }
5054 5146
5055 5147 mutex_exit(SQLOCK(sq));
5056 5148 }
5057 5149
5058 5150 /*
5059 5151 * Reset a flag that was set with blocksq.
5060 5152 *
5061 5153 * Can not use this routine to reset SQ_WRITER.
5062 5154 *
5063 5155 * If "isouter" is set then the syncq is assumed to be an outer perimeter
5064 5156 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5065 5157 * to handle the queued qwriter operations.
5066 5158 *
5067 5159 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5068 5160 * sq_putlocks are used.
5069 5161 */
5070 5162 static void
5071 5163 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5072 5164 {
5073 5165 uint16_t flags;
5074 5166
5075 5167 mutex_enter(SQLOCK(sq));
5076 5168 ASSERT(resetflag != SQ_WRITER);
5077 5169 ASSERT(sq->sq_flags & resetflag);
5078 5170 flags = sq->sq_flags & ~resetflag;
5079 5171 sq->sq_flags = flags;
5080 5172 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5081 5173 if (flags & SQ_WANTWAKEUP) {
5082 5174 flags &= ~SQ_WANTWAKEUP;
5083 5175 cv_broadcast(&sq->sq_wait);
5084 5176 }
5085 5177 sq->sq_flags = flags;
5086 5178 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5087 5179 if (!isouter) {
5088 5180 /* drain_syncq drops SQLOCK */
5089 5181 drain_syncq(sq);
5090 5182 return;
5091 5183 }
5092 5184 }
5093 5185 }
5094 5186 mutex_exit(SQLOCK(sq));
5095 5187 }
5096 5188
5097 5189 /*
5098 5190 * Reset a flag that was set with blocksq.
5099 5191 * Does not drain the syncq. Use emptysq() for that.
5100 5192 * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5101 5193 *
5102 5194 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5103 5195 * sq_putlocks are used.
5104 5196 */
5105 5197 static int
5106 5198 dropsq(syncq_t *sq, uint16_t resetflag)
5107 5199 {
5108 5200 uint16_t flags;
5109 5201
5110 5202 mutex_enter(SQLOCK(sq));
5111 5203 ASSERT(sq->sq_flags & resetflag);
5112 5204 flags = sq->sq_flags & ~resetflag;
5113 5205 if (flags & SQ_WANTWAKEUP) {
5114 5206 flags &= ~SQ_WANTWAKEUP;
5115 5207 cv_broadcast(&sq->sq_wait);
5116 5208 }
5117 5209 sq->sq_flags = flags;
5118 5210 mutex_exit(SQLOCK(sq));
5119 5211 if (flags & SQ_QUEUED)
5120 5212 return (1);
5121 5213 return (0);
5122 5214 }
5123 5215
5124 5216 /*
5125 5217 * Empty all the messages on a syncq.
5126 5218 *
5127 5219 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5128 5220 * sq_putlocks are used.
5129 5221 */
5130 5222 static void
5131 5223 emptysq(syncq_t *sq)
5132 5224 {
5133 5225 uint16_t flags;
5134 5226
5135 5227 mutex_enter(SQLOCK(sq));
5136 5228 flags = sq->sq_flags;
5137 5229 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5138 5230 /*
5139 5231 * To prevent potential recursive invocation of drain_syncq we
5140 5232 * do not call drain_syncq if count is non-zero.
5141 5233 */
5142 5234 if (sq->sq_count == 0) {
5143 5235 /* drain_syncq() drops SQLOCK */
5144 5236 drain_syncq(sq);
5145 5237 return;
5146 5238 } else
5147 5239 sqenable(sq);
5148 5240 }
5149 5241 mutex_exit(SQLOCK(sq));
5150 5242 }
5151 5243
5152 5244 /*
5153 5245 * Ordered insert while removing duplicates.
5154 5246 */
5155 5247 static void
5156 5248 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5157 5249 {
5158 5250 syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5159 5251
5160 5252 prev_sqlpp = &sqlist->sqlist_head;
5161 5253 while ((sqlp = *prev_sqlpp) != NULL) {
5162 5254 if (sqlp->sql_sq >= sqp) {
5163 5255 if (sqlp->sql_sq == sqp) /* duplicate */
5164 5256 return;
5165 5257 break;
5166 5258 }
5167 5259 prev_sqlpp = &sqlp->sql_next;
5168 5260 }
5169 5261 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5170 5262 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5171 5263 new_sqlp->sql_next = sqlp;
5172 5264 new_sqlp->sql_sq = sqp;
5173 5265 *prev_sqlpp = new_sqlp;
5174 5266 }
5175 5267
5176 5268 /*
5177 5269 * Walk the write side queues until we hit either the driver
5178 5270 * or a twist in the stream (_SAMESTR will return false in both
5179 5271 * these cases) then turn around and walk the read side queues
5180 5272 * back up to the stream head.
5181 5273 */
5182 5274 static void
5183 5275 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5184 5276 {
5185 5277 while (q != NULL) {
5186 5278 sqlist_insert(sqlist, q->q_syncq);
5187 5279
5188 5280 if (_SAMESTR(q))
5189 5281 q = q->q_next;
5190 5282 else if (!(q->q_flag & QREADR))
5191 5283 q = _RD(q);
5192 5284 else
5193 5285 q = NULL;
5194 5286 }
5195 5287 }
5196 5288
5197 5289 /*
5198 5290 * Allocate and build a list of all syncqs in a stream and the syncq(s)
5199 5291 * associated with the "q" parameter. The resulting list is sorted in a
5200 5292 * canonical order and is free of duplicates.
5201 5293 * Assumes the passed queue is a _RD(q).
5202 5294 */
5203 5295 static sqlist_t *
5204 5296 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5205 5297 {
5206 5298 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5207 5299
5208 5300 /*
5209 5301 * start with the current queue/qpair
5210 5302 */
5211 5303 ASSERT(q->q_flag & QREADR);
5212 5304
5213 5305 sqlist_insert(sqlist, q->q_syncq);
5214 5306 sqlist_insert(sqlist, _WR(q)->q_syncq);
5215 5307
5216 5308 sqlist_insertall(sqlist, stp->sd_wrq);
5217 5309 if (do_twist)
5218 5310 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5219 5311
5220 5312 return (sqlist);
5221 5313 }
5222 5314
5223 5315 static sqlist_t *
5224 5316 sqlist_alloc(struct stdata *stp, int kmflag)
5225 5317 {
5226 5318 size_t sqlist_size;
5227 5319 sqlist_t *sqlist;
5228 5320
5229 5321 /*
5230 5322 * Allocate 2 syncql_t's for each pushed module. Note that
5231 5323 * the sqlist_t structure already has 4 syncql_t's built in:
5232 5324 * 2 for the stream head, and 2 for the driver/other stream head.
5233 5325 */
5234 5326 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5235 5327 sizeof (sqlist_t);
5236 5328 if (STRMATED(stp))
5237 5329 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5238 5330 sqlist = kmem_alloc(sqlist_size, kmflag);
5239 5331
5240 5332 sqlist->sqlist_head = NULL;
5241 5333 sqlist->sqlist_size = sqlist_size;
5242 5334 sqlist->sqlist_index = 0;
5243 5335
5244 5336 return (sqlist);
5245 5337 }
5246 5338
5247 5339 /*
5248 5340 * Free the list created by sqlist_alloc()
5249 5341 */
5250 5342 static void
5251 5343 sqlist_free(sqlist_t *sqlist)
5252 5344 {
5253 5345 kmem_free(sqlist, sqlist->sqlist_size);
5254 5346 }
5255 5347
5256 5348 /*
5257 5349 * Prevent any new entries into any syncq in this stream.
5258 5350 * Used by freezestr.
5259 5351 */
5260 5352 void
5261 5353 strblock(queue_t *q)
5262 5354 {
5263 5355 struct stdata *stp;
5264 5356 syncql_t *sql;
5265 5357 sqlist_t *sqlist;
5266 5358
5267 5359 q = _RD(q);
5268 5360
5269 5361 stp = STREAM(q);
5270 5362 ASSERT(stp != NULL);
5271 5363
5272 5364 /*
5273 5365 * Get a sorted list with all the duplicates removed containing
5274 5366 * all the syncqs referenced by this stream.
5275 5367 */
5276 5368 sqlist = sqlist_build(q, stp, B_FALSE);
5277 5369 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5278 5370 blocksq(sql->sql_sq, SQ_FROZEN, -1);
5279 5371 sqlist_free(sqlist);
5280 5372 }
5281 5373
5282 5374 /*
5283 5375 * Release the block on new entries into this stream
5284 5376 */
5285 5377 void
5286 5378 strunblock(queue_t *q)
5287 5379 {
5288 5380 struct stdata *stp;
5289 5381 syncql_t *sql;
5290 5382 sqlist_t *sqlist;
5291 5383 int drain_needed;
5292 5384
5293 5385 q = _RD(q);
5294 5386
5295 5387 /*
5296 5388 * Get a sorted list with all the duplicates removed containing
5297 5389 * all the syncqs referenced by this stream.
5298 5390 * Have to drop the SQ_FROZEN flag on all the syncqs before
5299 5391 * starting to drain them; otherwise the draining might
5300 5392 * cause a freezestr in some module on the stream (which
5301 5393 * would deadlock).
5302 5394 */
5303 5395 stp = STREAM(q);
5304 5396 ASSERT(stp != NULL);
5305 5397 sqlist = sqlist_build(q, stp, B_FALSE);
5306 5398 drain_needed = 0;
5307 5399 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5308 5400 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5309 5401 if (drain_needed) {
5310 5402 for (sql = sqlist->sqlist_head; sql != NULL;
5311 5403 sql = sql->sql_next)
5312 5404 emptysq(sql->sql_sq);
5313 5405 }
5314 5406 sqlist_free(sqlist);
5315 5407 }
5316 5408
5317 5409 #ifdef DEBUG
5318 5410 static int
5319 5411 qprocsareon(queue_t *rq)
5320 5412 {
5321 5413 if (rq->q_next == NULL)
5322 5414 return (0);
5323 5415 return (_WR(rq->q_next)->q_next == _WR(rq));
5324 5416 }
5325 5417
5326 5418 int
5327 5419 qclaimed(queue_t *q)
5328 5420 {
5329 5421 uint_t count;
5330 5422
5331 5423 count = q->q_syncq->sq_count;
5332 5424 SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5333 5425 return (count != 0);
5334 5426 }
5335 5427
5336 5428 /*
5337 5429 * Check if anyone has frozen this stream with freezestr
5338 5430 */
5339 5431 int
5340 5432 frozenstr(queue_t *q)
5341 5433 {
5342 5434 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5343 5435 }
5344 5436 #endif /* DEBUG */
5345 5437
5346 5438 /*
5347 5439 * Enter a queue.
5348 5440 * Obsoleted interface. Should not be used.
5349 5441 */
5350 5442 void
5351 5443 enterq(queue_t *q)
5352 5444 {
5353 5445 entersq(q->q_syncq, SQ_CALLBACK);
5354 5446 }
5355 5447
5356 5448 void
5357 5449 leaveq(queue_t *q)
5358 5450 {
5359 5451 leavesq(q->q_syncq, SQ_CALLBACK);
5360 5452 }
5361 5453
5362 5454 /*
5363 5455 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5364 5456 * to check.
5365 5457 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5366 5458 * calls and the running of open, close and service procedures.
5367 5459 *
5368 5460 * If c_inner bit is set no need to grab sq_putlocks since we don't care
5369 5461 * if other threads have entered or are entering put entry point.
5370 5462 *
5371 5463 * If c_inner bit is set it might have been possible to use
5372 5464 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5373 5465 * open/close path for IP) but since the count may need to be decremented in
5374 5466 * qwait() we wouldn't know which counter to decrement. Currently counter is
5375 5467 * selected by current cpu_seqid and current CPU can change at any moment. XXX
5376 5468 * in the future we might use curthread id bits to select the counter and this
5377 5469 * would stay constant across routine calls.
5378 5470 */
5379 5471 void
5380 5472 entersq(syncq_t *sq, int entrypoint)
5381 5473 {
5382 5474 uint16_t count = 0;
5383 5475 uint16_t flags;
5384 5476 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5385 5477 uint16_t type;
5386 5478 uint_t c_inner = entrypoint & SQ_CI;
5387 5479 uint_t c_outer = entrypoint & SQ_CO;
5388 5480
5389 5481 /*
5390 5482 * Increment ref count to keep closes out of this queue.
5391 5483 */
5392 5484 ASSERT(sq);
5393 5485 ASSERT(c_inner && c_outer);
5394 5486 mutex_enter(SQLOCK(sq));
5395 5487 flags = sq->sq_flags;
5396 5488 type = sq->sq_type;
5397 5489 if (!(type & c_inner)) {
5398 5490 /* Make sure all putcounts now use slowlock. */
5399 5491 count = sq->sq_count;
5400 5492 SQ_PUTLOCKS_ENTER(sq);
5401 5493 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5402 5494 SUM_SQ_PUTCOUNTS(sq, count);
5403 5495 sq->sq_needexcl++;
5404 5496 ASSERT(sq->sq_needexcl != 0); /* wraparound */
5405 5497 waitflags |= SQ_MESSAGES;
5406 5498 }
5407 5499 /*
5408 5500 * Wait until we can enter the inner perimeter.
5409 5501 * If we want exclusive access we wait until sq_count is 0.
5410 5502 * We have to do this before entering the outer perimeter in order
5411 5503 * to preserve put/close message ordering.
5412 5504 */
5413 5505 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5414 5506 sq->sq_flags = flags | SQ_WANTWAKEUP;
5415 5507 if (!(type & c_inner)) {
5416 5508 SQ_PUTLOCKS_EXIT(sq);
5417 5509 }
5418 5510 cv_wait(&sq->sq_wait, SQLOCK(sq));
5419 5511 if (!(type & c_inner)) {
5420 5512 count = sq->sq_count;
5421 5513 SQ_PUTLOCKS_ENTER(sq);
5422 5514 SUM_SQ_PUTCOUNTS(sq, count);
5423 5515 }
5424 5516 flags = sq->sq_flags;
5425 5517 }
5426 5518
5427 5519 if (!(type & c_inner)) {
5428 5520 ASSERT(sq->sq_needexcl > 0);
5429 5521 sq->sq_needexcl--;
5430 5522 if (sq->sq_needexcl == 0) {
5431 5523 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5432 5524 }
5433 5525 }
5434 5526
5435 5527 /* Check if we need to enter the outer perimeter */
5436 5528 if (!(type & c_outer)) {
5437 5529 /*
5438 5530 * We have to enter the outer perimeter exclusively before
5439 5531 * we can increment sq_count to avoid deadlock. This implies
5440 5532 * that we have to re-check sq_flags and sq_count.
5441 5533 *
5442 5534 * is it possible to have c_inner set when c_outer is not set?
5443 5535 */
5444 5536 if (!(type & c_inner)) {
5445 5537 SQ_PUTLOCKS_EXIT(sq);
5446 5538 }
5447 5539 mutex_exit(SQLOCK(sq));
5448 5540 outer_enter(sq->sq_outer, SQ_GOAWAY);
5449 5541 mutex_enter(SQLOCK(sq));
5450 5542 flags = sq->sq_flags;
5451 5543 /*
5452 5544 * there should be no need to recheck sq_putcounts
5453 5545 * because outer_enter() has already waited for them to clear
5454 5546 * after setting SQ_WRITER.
5455 5547 */
5456 5548 count = sq->sq_count;
5457 5549 #ifdef DEBUG
5458 5550 /*
5459 5551 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5460 5552 * of doing an ASSERT internally. Others should do
5461 5553 * something like
5462 5554 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5463 5555 * without the need to #ifdef DEBUG it.
5464 5556 */
5465 5557 SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5466 5558 #endif
5467 5559 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5468 5560 (!(type & c_inner) && count != 0)) {
5469 5561 sq->sq_flags = flags | SQ_WANTWAKEUP;
5470 5562 cv_wait(&sq->sq_wait, SQLOCK(sq));
5471 5563 count = sq->sq_count;
5472 5564 flags = sq->sq_flags;
5473 5565 }
5474 5566 }
5475 5567
5476 5568 sq->sq_count++;
5477 5569 ASSERT(sq->sq_count != 0); /* Wraparound */
5478 5570 if (!(type & c_inner)) {
5479 5571 /* Exclusive entry */
5480 5572 ASSERT(sq->sq_count == 1);
5481 5573 sq->sq_flags |= SQ_EXCL;
5482 5574 if (type & c_outer) {
5483 5575 SQ_PUTLOCKS_EXIT(sq);
5484 5576 }
5485 5577 }
5486 5578 mutex_exit(SQLOCK(sq));
5487 5579 }
5488 5580
5489 5581 /*
5490 5582 * Leave a syncq. Announce to framework that closes may proceed.
5491 5583 * c_inner and c_outer specify which concurrency bits to check.
5492 5584 *
5493 5585 * Must never be called from driver or module put entry point.
5494 5586 *
5495 5587 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5496 5588 * sq_putlocks are used.
5497 5589 */
5498 5590 void
5499 5591 leavesq(syncq_t *sq, int entrypoint)
5500 5592 {
5501 5593 uint16_t flags;
5502 5594 uint16_t type;
5503 5595 uint_t c_outer = entrypoint & SQ_CO;
5504 5596 #ifdef DEBUG
5505 5597 uint_t c_inner = entrypoint & SQ_CI;
5506 5598 #endif
5507 5599
5508 5600 /*
5509 5601 * Decrement ref count, drain the syncq if possible, and wake up
5510 5602 * any waiting close.
5511 5603 */
5512 5604 ASSERT(sq);
5513 5605 ASSERT(c_inner && c_outer);
5514 5606 mutex_enter(SQLOCK(sq));
5515 5607 flags = sq->sq_flags;
5516 5608 type = sq->sq_type;
5517 5609 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5518 5610
5519 5611 if (flags & SQ_WANTWAKEUP) {
5520 5612 flags &= ~SQ_WANTWAKEUP;
5521 5613 cv_broadcast(&sq->sq_wait);
5522 5614 }
5523 5615 if (flags & SQ_WANTEXWAKEUP) {
5524 5616 flags &= ~SQ_WANTEXWAKEUP;
5525 5617 cv_broadcast(&sq->sq_exitwait);
5526 5618 }
5527 5619
5528 5620 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5529 5621 /*
5530 5622 * The syncq needs to be drained. "Exit" the syncq
5531 5623 * before calling drain_syncq.
5532 5624 */
5533 5625 ASSERT(sq->sq_count != 0);
5534 5626 sq->sq_count--;
5535 5627 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5536 5628 sq->sq_flags = flags & ~SQ_EXCL;
5537 5629 drain_syncq(sq);
5538 5630 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5539 5631 /* Check if we need to exit the outer perimeter */
5540 5632 /* XXX will this ever be true? */
5541 5633 if (!(type & c_outer))
5542 5634 outer_exit(sq->sq_outer);
5543 5635 return;
5544 5636 }
5545 5637 }
5546 5638 ASSERT(sq->sq_count != 0);
5547 5639 sq->sq_count--;
5548 5640 ASSERT((flags & SQ_EXCL) || (type & c_inner));
5549 5641 sq->sq_flags = flags & ~SQ_EXCL;
5550 5642 mutex_exit(SQLOCK(sq));
5551 5643
5552 5644 /* Check if we need to exit the outer perimeter */
5553 5645 if (!(sq->sq_type & c_outer))
5554 5646 outer_exit(sq->sq_outer);
5555 5647 }
5556 5648
5557 5649 /*
5558 5650 * Prevent q_next from changing in this stream by incrementing sq_count.
5559 5651 *
5560 5652 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5561 5653 * sq_putlocks are used.
5562 5654 */
5563 5655 void
5564 5656 claimq(queue_t *qp)
5565 5657 {
5566 5658 syncq_t *sq = qp->q_syncq;
5567 5659
5568 5660 mutex_enter(SQLOCK(sq));
5569 5661 sq->sq_count++;
5570 5662 ASSERT(sq->sq_count != 0); /* Wraparound */
5571 5663 mutex_exit(SQLOCK(sq));
5572 5664 }
5573 5665
5574 5666 /*
5575 5667 * Undo claimq.
5576 5668 *
5577 5669 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5578 5670 * sq_putlocks are used.
5579 5671 */
5580 5672 void
5581 5673 releaseq(queue_t *qp)
5582 5674 {
5583 5675 syncq_t *sq = qp->q_syncq;
5584 5676 uint16_t flags;
5585 5677
5586 5678 mutex_enter(SQLOCK(sq));
5587 5679 ASSERT(sq->sq_count > 0);
5588 5680 sq->sq_count--;
5589 5681
5590 5682 flags = sq->sq_flags;
5591 5683 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5592 5684 if (flags & SQ_WANTWAKEUP) {
5593 5685 flags &= ~SQ_WANTWAKEUP;
5594 5686 cv_broadcast(&sq->sq_wait);
5595 5687 }
5596 5688 sq->sq_flags = flags;
5597 5689 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5598 5690 /*
5599 5691 * To prevent potential recursive invocation of
5600 5692 * drain_syncq we do not call drain_syncq if count is
5601 5693 * non-zero.
5602 5694 */
5603 5695 if (sq->sq_count == 0) {
5604 5696 drain_syncq(sq);
5605 5697 return;
5606 5698 } else
5607 5699 sqenable(sq);
5608 5700 }
5609 5701 }
5610 5702 mutex_exit(SQLOCK(sq));
5611 5703 }
5612 5704
5613 5705 /*
5614 5706 * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5615 5707 */
5616 5708 void
5617 5709 claimstr(queue_t *qp)
5618 5710 {
5619 5711 struct stdata *stp = STREAM(qp);
5620 5712
5621 5713 mutex_enter(&stp->sd_reflock);
5622 5714 stp->sd_refcnt++;
5623 5715 ASSERT(stp->sd_refcnt != 0); /* Wraparound */
5624 5716 mutex_exit(&stp->sd_reflock);
5625 5717 }
5626 5718
5627 5719 /*
5628 5720 * Undo claimstr.
5629 5721 */
5630 5722 void
5631 5723 releasestr(queue_t *qp)
5632 5724 {
5633 5725 struct stdata *stp = STREAM(qp);
5634 5726
5635 5727 mutex_enter(&stp->sd_reflock);
5636 5728 ASSERT(stp->sd_refcnt != 0);
5637 5729 if (--stp->sd_refcnt == 0)
5638 5730 cv_broadcast(&stp->sd_refmonitor);
5639 5731 mutex_exit(&stp->sd_reflock);
5640 5732 }
5641 5733
5642 5734 static syncq_t *
5643 5735 new_syncq(void)
5644 5736 {
5645 5737 return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5646 5738 }
5647 5739
5648 5740 static void
5649 5741 free_syncq(syncq_t *sq)
5650 5742 {
5651 5743 ASSERT(sq->sq_head == NULL);
5652 5744 ASSERT(sq->sq_outer == NULL);
5653 5745 ASSERT(sq->sq_callbpend == NULL);
5654 5746 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5655 5747 (sq->sq_onext == sq && sq->sq_oprev == sq));
5656 5748
5657 5749 if (sq->sq_ciputctrl != NULL) {
5658 5750 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5659 5751 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5660 5752 sq->sq_nciputctrl, 0);
5661 5753 ASSERT(ciputctrl_cache != NULL);
5662 5754 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5663 5755 }
5664 5756
5665 5757 sq->sq_tail = NULL;
5666 5758 sq->sq_evhead = NULL;
5667 5759 sq->sq_evtail = NULL;
5668 5760 sq->sq_ciputctrl = NULL;
5669 5761 sq->sq_nciputctrl = 0;
5670 5762 sq->sq_count = 0;
5671 5763 sq->sq_rmqcount = 0;
5672 5764 sq->sq_callbflags = 0;
5673 5765 sq->sq_cancelid = 0;
5674 5766 sq->sq_next = NULL;
5675 5767 sq->sq_needexcl = 0;
5676 5768 sq->sq_svcflags = 0;
5677 5769 sq->sq_nqueues = 0;
5678 5770 sq->sq_pri = 0;
5679 5771 sq->sq_onext = NULL;
5680 5772 sq->sq_oprev = NULL;
5681 5773 sq->sq_flags = 0;
5682 5774 sq->sq_type = 0;
5683 5775 sq->sq_servcount = 0;
5684 5776
5685 5777 kmem_cache_free(syncq_cache, sq);
5686 5778 }
5687 5779
5688 5780 /* Outer perimeter code */
5689 5781
5690 5782 /*
5691 5783 * The outer syncq uses the fields and flags in the syncq slightly
5692 5784 * differently from the inner syncqs.
5693 5785 * sq_count Incremented when there are pending or running
5694 5786 * writers at the outer perimeter to prevent the set of
5695 5787 * inner syncqs that belong to the outer perimeter from
5696 5788 * changing.
5697 5789 * sq_head/tail List of deferred qwriter(OUTER) operations.
5698 5790 *
5699 5791 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while
5700 5792 * inner syncqs are added to or removed from the
5701 5793 * outer perimeter.
5702 5794 * SQ_QUEUED sq_head/tail has messages or events queued.
5703 5795 *
5704 5796 * SQ_WRITER A thread is currently traversing all the inner syncqs
5705 5797 * setting the SQ_WRITER flag.
5706 5798 */
5707 5799
5708 5800 /*
5709 5801 * Get write access at the outer perimeter.
5710 5802 * Note that read access is done by entersq, putnext, and put by simply
5711 5803 * incrementing sq_count in the inner syncq.
5712 5804 *
5713 5805 * Waits until "flags" is no longer set in the outer to prevent multiple
5714 5806 * threads from having write access at the same time. SQ_WRITER has to be part
5715 5807 * of "flags".
5716 5808 *
5717 5809 * Increases sq_count on the outer syncq to keep away outer_insert/remove
5718 5810 * until the outer_exit is finished.
5719 5811 *
5720 5812 * outer_enter is vulnerable to starvation since it does not prevent new
5721 5813 * threads from entering the inner syncqs while it is waiting for sq_count to
5722 5814 * go to zero.
5723 5815 */
5724 5816 void
5725 5817 outer_enter(syncq_t *outer, uint16_t flags)
5726 5818 {
5727 5819 syncq_t *sq;
5728 5820 int wait_needed;
5729 5821 uint16_t count;
5730 5822
5731 5823 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5732 5824 outer->sq_oprev != NULL);
5733 5825 ASSERT(flags & SQ_WRITER);
5734 5826
5735 5827 retry:
5736 5828 mutex_enter(SQLOCK(outer));
5737 5829 while (outer->sq_flags & flags) {
5738 5830 outer->sq_flags |= SQ_WANTWAKEUP;
5739 5831 cv_wait(&outer->sq_wait, SQLOCK(outer));
5740 5832 }
5741 5833
5742 5834 ASSERT(!(outer->sq_flags & SQ_WRITER));
5743 5835 outer->sq_flags |= SQ_WRITER;
5744 5836 outer->sq_count++;
5745 5837 ASSERT(outer->sq_count != 0); /* wraparound */
5746 5838 wait_needed = 0;
5747 5839 /*
5748 5840 * Set SQ_WRITER on all the inner syncqs while holding
5749 5841 * the SQLOCK on the outer syncq. This ensures that the changing
5750 5842 * of SQ_WRITER is atomic under the outer SQLOCK.
5751 5843 */
5752 5844 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5753 5845 mutex_enter(SQLOCK(sq));
5754 5846 count = sq->sq_count;
5755 5847 SQ_PUTLOCKS_ENTER(sq);
5756 5848 sq->sq_flags |= SQ_WRITER;
5757 5849 SUM_SQ_PUTCOUNTS(sq, count);
5758 5850 if (count != 0)
5759 5851 wait_needed = 1;
5760 5852 SQ_PUTLOCKS_EXIT(sq);
5761 5853 mutex_exit(SQLOCK(sq));
5762 5854 }
5763 5855 mutex_exit(SQLOCK(outer));
5764 5856
5765 5857 /*
5766 5858 * Get everybody out of the syncqs sequentially.
5767 5859 * Note that we don't actually need to acquire the PUTLOCKS, since
5768 5860 * we have already cleared the fastbit, and set QWRITER. By
5769 5861 * definition, the count can not increase since putnext will
5770 5862 * take the slowlock path (and the purpose of acquiring the
5771 5863 * putlocks was to make sure it didn't increase while we were
5772 5864 * waiting).
5773 5865 *
5774 5866 * Note that we still acquire the PUTLOCKS to be safe.
5775 5867 */
5776 5868 if (wait_needed) {
5777 5869 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5778 5870 mutex_enter(SQLOCK(sq));
5779 5871 count = sq->sq_count;
5780 5872 SQ_PUTLOCKS_ENTER(sq);
5781 5873 SUM_SQ_PUTCOUNTS(sq, count);
5782 5874 while (count != 0) {
5783 5875 sq->sq_flags |= SQ_WANTWAKEUP;
5784 5876 SQ_PUTLOCKS_EXIT(sq);
5785 5877 cv_wait(&sq->sq_wait, SQLOCK(sq));
5786 5878 count = sq->sq_count;
5787 5879 SQ_PUTLOCKS_ENTER(sq);
5788 5880 SUM_SQ_PUTCOUNTS(sq, count);
5789 5881 }
5790 5882 SQ_PUTLOCKS_EXIT(sq);
5791 5883 mutex_exit(SQLOCK(sq));
5792 5884 }
5793 5885 /*
5794 5886 * Verify that none of the flags got set while we
5795 5887 * were waiting for the sq_counts to drop.
5796 5888 * If this happens we exit and retry entering the
5797 5889 * outer perimeter.
5798 5890 */
5799 5891 mutex_enter(SQLOCK(outer));
5800 5892 if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5801 5893 mutex_exit(SQLOCK(outer));
5802 5894 outer_exit(outer);
5803 5895 goto retry;
5804 5896 }
5805 5897 mutex_exit(SQLOCK(outer));
5806 5898 }
5807 5899 }
5808 5900
5809 5901 /*
5810 5902 * Drop the write access at the outer perimeter.
5811 5903 * Read access is dropped implicitly (by putnext, put, and leavesq) by
5812 5904 * decrementing sq_count.
5813 5905 */
5814 5906 void
5815 5907 outer_exit(syncq_t *outer)
5816 5908 {
5817 5909 syncq_t *sq;
5818 5910 int drain_needed;
5819 5911 uint16_t flags;
5820 5912
5821 5913 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5822 5914 outer->sq_oprev != NULL);
5823 5915 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5824 5916
5825 5917 /*
5826 5918 * Atomically (from the perspective of threads calling become_writer)
5827 5919 * drop the write access at the outer perimeter by holding
5828 5920 * SQLOCK(outer) across all the dropsq calls and the resetting of
5829 5921 * SQ_WRITER.
5830 5922 * This defines a locking order between the outer perimeter
5831 5923 * SQLOCK and the inner perimeter SQLOCKs.
5832 5924 */
5833 5925 mutex_enter(SQLOCK(outer));
5834 5926 flags = outer->sq_flags;
5835 5927 ASSERT(outer->sq_flags & SQ_WRITER);
5836 5928 if (flags & SQ_QUEUED) {
5837 5929 write_now(outer);
5838 5930 flags = outer->sq_flags;
5839 5931 }
5840 5932
5841 5933 /*
5842 5934 * sq_onext is stable since sq_count has not yet been decreased.
5843 5935 * Reset the SQ_WRITER flags in all syncqs.
5844 5936 * After dropping SQ_WRITER on the outer syncq we empty all the
5845 5937 * inner syncqs.
5846 5938 */
5847 5939 drain_needed = 0;
5848 5940 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5849 5941 drain_needed += dropsq(sq, SQ_WRITER);
5850 5942 ASSERT(!(outer->sq_flags & SQ_QUEUED));
5851 5943 flags &= ~SQ_WRITER;
5852 5944 if (drain_needed) {
5853 5945 outer->sq_flags = flags;
5854 5946 mutex_exit(SQLOCK(outer));
5855 5947 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5856 5948 emptysq(sq);
5857 5949 mutex_enter(SQLOCK(outer));
5858 5950 flags = outer->sq_flags;
5859 5951 }
5860 5952 if (flags & SQ_WANTWAKEUP) {
5861 5953 flags &= ~SQ_WANTWAKEUP;
5862 5954 cv_broadcast(&outer->sq_wait);
5863 5955 }
5864 5956 outer->sq_flags = flags;
5865 5957 ASSERT(outer->sq_count > 0);
5866 5958 outer->sq_count--;
5867 5959 mutex_exit(SQLOCK(outer));
5868 5960 }
5869 5961
5870 5962 /*
5871 5963 * Add another syncq to an outer perimeter.
5872 5964 * Block out all other access to the outer perimeter while it is being
5873 5965 * changed using blocksq.
5874 5966 * Assumes that the caller has *not* done an outer_enter.
5875 5967 *
5876 5968 * Vulnerable to starvation in blocksq.
5877 5969 */
5878 5970 static void
5879 5971 outer_insert(syncq_t *outer, syncq_t *sq)
5880 5972 {
5881 5973 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5882 5974 outer->sq_oprev != NULL);
5883 5975 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5884 5976 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */
5885 5977
5886 5978 /* Get exclusive access to the outer perimeter list */
5887 5979 blocksq(outer, SQ_BLOCKED, 0);
5888 5980 ASSERT(outer->sq_flags & SQ_BLOCKED);
5889 5981 ASSERT(!(outer->sq_flags & SQ_WRITER));
5890 5982
5891 5983 mutex_enter(SQLOCK(sq));
5892 5984 sq->sq_outer = outer;
5893 5985 outer->sq_onext->sq_oprev = sq;
5894 5986 sq->sq_onext = outer->sq_onext;
5895 5987 outer->sq_onext = sq;
5896 5988 sq->sq_oprev = outer;
5897 5989 mutex_exit(SQLOCK(sq));
5898 5990 unblocksq(outer, SQ_BLOCKED, 1);
5899 5991 }
5900 5992
5901 5993 /*
5902 5994 * Remove a syncq from an outer perimeter.
5903 5995 * Block out all other access to the outer perimeter while it is being
5904 5996 * changed using blocksq.
5905 5997 * Assumes that the caller has *not* done an outer_enter.
5906 5998 *
5907 5999 * Vulnerable to starvation in blocksq.
5908 6000 */
5909 6001 static void
5910 6002 outer_remove(syncq_t *outer, syncq_t *sq)
5911 6003 {
5912 6004 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5913 6005 outer->sq_oprev != NULL);
5914 6006 ASSERT(sq->sq_outer == outer);
5915 6007
5916 6008 /* Get exclusive access to the outer perimeter list */
5917 6009 blocksq(outer, SQ_BLOCKED, 0);
5918 6010 ASSERT(outer->sq_flags & SQ_BLOCKED);
5919 6011 ASSERT(!(outer->sq_flags & SQ_WRITER));
5920 6012
5921 6013 mutex_enter(SQLOCK(sq));
5922 6014 sq->sq_outer = NULL;
5923 6015 sq->sq_onext->sq_oprev = sq->sq_oprev;
5924 6016 sq->sq_oprev->sq_onext = sq->sq_onext;
5925 6017 sq->sq_oprev = sq->sq_onext = NULL;
5926 6018 mutex_exit(SQLOCK(sq));
5927 6019 unblocksq(outer, SQ_BLOCKED, 1);
5928 6020 }
5929 6021
5930 6022 /*
5931 6023 * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5932 6024 * If this is the first callback for this outer perimeter then add
5933 6025 * this outer perimeter to the list of outer perimeters that
5934 6026 * the qwriter_outer_thread will process.
5935 6027 *
5936 6028 * Increments sq_count in the outer syncq to prevent the membership
5937 6029 * of the outer perimeter (in terms of inner syncqs) to change while
5938 6030 * the callback is pending.
5939 6031 */
5940 6032 static void
5941 6033 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5942 6034 {
5943 6035 ASSERT(MUTEX_HELD(SQLOCK(outer)));
5944 6036
5945 6037 mp->b_prev = (mblk_t *)func;
5946 6038 mp->b_queue = q;
5947 6039 mp->b_next = NULL;
5948 6040 outer->sq_count++; /* Decremented when dequeued */
5949 6041 ASSERT(outer->sq_count != 0); /* Wraparound */
5950 6042 if (outer->sq_evhead == NULL) {
5951 6043 /* First message. */
5952 6044 outer->sq_evhead = outer->sq_evtail = mp;
5953 6045 outer->sq_flags |= SQ_EVENTS;
5954 6046 mutex_exit(SQLOCK(outer));
5955 6047 STRSTAT(qwr_outer);
5956 6048 (void) taskq_dispatch(streams_taskq,
5957 6049 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5958 6050 } else {
5959 6051 ASSERT(outer->sq_flags & SQ_EVENTS);
5960 6052 outer->sq_evtail->b_next = mp;
5961 6053 outer->sq_evtail = mp;
5962 6054 mutex_exit(SQLOCK(outer));
5963 6055 }
5964 6056 }
5965 6057
5966 6058 /*
5967 6059 * Try and upgrade to write access at the outer perimeter. If this can
5968 6060 * not be done without blocking then queue the callback to be done
5969 6061 * by the qwriter_outer_thread.
5970 6062 *
5971 6063 * This routine can only be called from put or service procedures plus
5972 6064 * asynchronous callback routines that have properly entered the queue (with
5973 6065 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5974 6066 * associated with q.
5975 6067 */
5976 6068 void
5977 6069 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5978 6070 {
5979 6071 syncq_t *osq, *sq, *outer;
5980 6072 int failed;
5981 6073 uint16_t flags;
5982 6074
5983 6075 osq = q->q_syncq;
5984 6076 outer = osq->sq_outer;
5985 6077 if (outer == NULL)
5986 6078 panic("qwriter(PERIM_OUTER): no outer perimeter");
5987 6079 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5988 6080 outer->sq_oprev != NULL);
5989 6081
5990 6082 mutex_enter(SQLOCK(outer));
5991 6083 flags = outer->sq_flags;
5992 6084 /*
5993 6085 * If some thread is traversing sq_next, or if we are blocked by
5994 6086 * outer_insert or outer_remove, or if the we already have queued
5995 6087 * callbacks, then queue this callback for later processing.
5996 6088 *
5997 6089 * Also queue the qwriter for an interrupt thread in order
5998 6090 * to reduce the time spent running at high IPL.
5999 6091 * to identify there are events.
6000 6092 */
6001 6093 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6002 6094 /*
6003 6095 * Queue the become_writer request.
6004 6096 * The queueing is atomic under SQLOCK(outer) in order
6005 6097 * to synchronize with outer_exit.
6006 6098 * queue_writer will drop the outer SQLOCK
6007 6099 */
6008 6100 if (flags & SQ_BLOCKED) {
6009 6101 /* Must set SQ_WRITER on inner perimeter */
6010 6102 mutex_enter(SQLOCK(osq));
6011 6103 osq->sq_flags |= SQ_WRITER;
6012 6104 mutex_exit(SQLOCK(osq));
6013 6105 } else {
6014 6106 if (!(flags & SQ_WRITER)) {
6015 6107 /*
6016 6108 * The outer could have been SQ_BLOCKED thus
6017 6109 * SQ_WRITER might not be set on the inner.
6018 6110 */
6019 6111 mutex_enter(SQLOCK(osq));
6020 6112 osq->sq_flags |= SQ_WRITER;
6021 6113 mutex_exit(SQLOCK(osq));
6022 6114 }
6023 6115 ASSERT(osq->sq_flags & SQ_WRITER);
6024 6116 }
6025 6117 queue_writer(outer, func, q, mp);
6026 6118 return;
6027 6119 }
6028 6120 /*
6029 6121 * We are half-way to exclusive access to the outer perimeter.
6030 6122 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6031 6123 * while the inner syncqs are traversed.
6032 6124 */
6033 6125 outer->sq_count++;
6034 6126 ASSERT(outer->sq_count != 0); /* wraparound */
6035 6127 flags |= SQ_WRITER;
6036 6128 /*
6037 6129 * Check if we can run the function immediately. Mark all
6038 6130 * syncqs with the writer flag to prevent new entries into
6039 6131 * put and service procedures.
6040 6132 *
6041 6133 * Set SQ_WRITER on all the inner syncqs while holding
6042 6134 * the SQLOCK on the outer syncq. This ensures that the changing
6043 6135 * of SQ_WRITER is atomic under the outer SQLOCK.
6044 6136 */
6045 6137 failed = 0;
6046 6138 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6047 6139 uint16_t count;
6048 6140 uint_t maxcnt = (sq == osq) ? 1 : 0;
6049 6141
6050 6142 mutex_enter(SQLOCK(sq));
6051 6143 count = sq->sq_count;
6052 6144 SQ_PUTLOCKS_ENTER(sq);
6053 6145 SUM_SQ_PUTCOUNTS(sq, count);
6054 6146 if (sq->sq_count > maxcnt)
6055 6147 failed = 1;
6056 6148 sq->sq_flags |= SQ_WRITER;
6057 6149 SQ_PUTLOCKS_EXIT(sq);
6058 6150 mutex_exit(SQLOCK(sq));
6059 6151 }
6060 6152 if (failed) {
6061 6153 /*
6062 6154 * Some other thread has a read claim on the outer perimeter.
6063 6155 * Queue the callback for deferred processing.
6064 6156 *
6065 6157 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6066 6158 * so that other qwriter(OUTER) calls will queue their
6067 6159 * callbacks as well. queue_writer increments sq_count so we
6068 6160 * decrement to compensate for the our increment.
6069 6161 *
6070 6162 * Dropping SQ_WRITER enables the writer thread to work
6071 6163 * on this outer perimeter.
6072 6164 */
6073 6165 outer->sq_flags = flags;
6074 6166 queue_writer(outer, func, q, mp);
6075 6167 /* queue_writer dropper the lock */
6076 6168 mutex_enter(SQLOCK(outer));
6077 6169 ASSERT(outer->sq_count > 0);
6078 6170 outer->sq_count--;
6079 6171 ASSERT(outer->sq_flags & SQ_WRITER);
6080 6172 flags = outer->sq_flags;
6081 6173 flags &= ~SQ_WRITER;
6082 6174 if (flags & SQ_WANTWAKEUP) {
6083 6175 flags &= ~SQ_WANTWAKEUP;
6084 6176 cv_broadcast(&outer->sq_wait);
6085 6177 }
6086 6178 outer->sq_flags = flags;
6087 6179 mutex_exit(SQLOCK(outer));
6088 6180 return;
6089 6181 } else {
6090 6182 outer->sq_flags = flags;
6091 6183 mutex_exit(SQLOCK(outer));
6092 6184 }
6093 6185
6094 6186 /* Can run it immediately */
6095 6187 (*func)(q, mp);
6096 6188
6097 6189 outer_exit(outer);
6098 6190 }
6099 6191
6100 6192 /*
6101 6193 * Dequeue all writer callbacks from the outer perimeter and run them.
6102 6194 */
6103 6195 static void
6104 6196 write_now(syncq_t *outer)
6105 6197 {
6106 6198 mblk_t *mp;
6107 6199 queue_t *q;
6108 6200 void (*func)();
6109 6201
6110 6202 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6111 6203 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6112 6204 outer->sq_oprev != NULL);
6113 6205 while ((mp = outer->sq_evhead) != NULL) {
6114 6206 /*
6115 6207 * queues cannot be placed on the queuelist on the outer
6116 6208 * perimeter.
6117 6209 */
6118 6210 ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6119 6211 ASSERT((outer->sq_flags & SQ_EVENTS));
6120 6212
6121 6213 outer->sq_evhead = mp->b_next;
6122 6214 if (outer->sq_evhead == NULL) {
6123 6215 outer->sq_evtail = NULL;
6124 6216 outer->sq_flags &= ~SQ_EVENTS;
6125 6217 }
6126 6218 ASSERT(outer->sq_count != 0);
6127 6219 outer->sq_count--; /* Incremented when enqueued. */
6128 6220 mutex_exit(SQLOCK(outer));
6129 6221 /*
6130 6222 * Drop the message if the queue is closing.
6131 6223 * Make sure that the queue is "claimed" when the callback
6132 6224 * is run in order to satisfy various ASSERTs.
6133 6225 */
6134 6226 q = mp->b_queue;
6135 6227 func = (void (*)())mp->b_prev;
6136 6228 ASSERT(func != NULL);
6137 6229 mp->b_next = mp->b_prev = NULL;
6138 6230 if (q->q_flag & QWCLOSE) {
6139 6231 freemsg(mp);
6140 6232 } else {
6141 6233 claimq(q);
6142 6234 (*func)(q, mp);
6143 6235 releaseq(q);
6144 6236 }
6145 6237 mutex_enter(SQLOCK(outer));
6146 6238 }
6147 6239 ASSERT(MUTEX_HELD(SQLOCK(outer)));
6148 6240 }
6149 6241
6150 6242 /*
6151 6243 * The list of messages on the inner syncq is effectively hashed
6152 6244 * by destination queue. These destination queues are doubly
6153 6245 * linked lists (hopefully) in priority order. Messages are then
6154 6246 * put on the queue referenced by the q_sqhead/q_sqtail elements.
6155 6247 * Additional messages are linked together by the b_next/b_prev
6156 6248 * elements in the mblk, with (similar to putq()) the first message
6157 6249 * having a NULL b_prev and the last message having a NULL b_next.
6158 6250 *
6159 6251 * Events, such as qwriter callbacks, are put onto a list in FIFO
6160 6252 * order referenced by sq_evhead, and sq_evtail. This is a singly
6161 6253 * linked list, and messages here MUST be processed in the order queued.
6162 6254 */
6163 6255
6164 6256 /*
6165 6257 * Run the events on the syncq event list (sq_evhead).
6166 6258 * Assumes there is only one claim on the syncq, it is
6167 6259 * already exclusive (SQ_EXCL set), and the SQLOCK held.
6168 6260 * Messages here are processed in order, with the SQ_EXCL bit
6169 6261 * held all the way through till the last message is processed.
6170 6262 */
6171 6263 void
6172 6264 sq_run_events(syncq_t *sq)
6173 6265 {
6174 6266 mblk_t *bp;
6175 6267 queue_t *qp;
6176 6268 uint16_t flags = sq->sq_flags;
6177 6269 void (*func)();
6178 6270
6179 6271 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6180 6272 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6181 6273 sq->sq_oprev == NULL) ||
6182 6274 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6183 6275 sq->sq_oprev != NULL));
6184 6276
6185 6277 ASSERT(flags & SQ_EXCL);
6186 6278 ASSERT(sq->sq_count == 1);
6187 6279
6188 6280 /*
6189 6281 * We need to process all of the events on this list. It
6190 6282 * is possible that new events will be added while we are
6191 6283 * away processing a callback, so on every loop, we start
6192 6284 * back at the beginning of the list.
6193 6285 */
6194 6286 /*
6195 6287 * We have to reaccess sq_evhead since there is a
6196 6288 * possibility of a new entry while we were running
6197 6289 * the callback.
6198 6290 */
6199 6291 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6200 6292 ASSERT(bp->b_queue->q_syncq == sq);
6201 6293 ASSERT(sq->sq_flags & SQ_EVENTS);
6202 6294
6203 6295 qp = bp->b_queue;
6204 6296 func = (void (*)())bp->b_prev;
6205 6297 ASSERT(func != NULL);
6206 6298
6207 6299 /*
6208 6300 * Messages from the event queue must be taken off in
6209 6301 * FIFO order.
6210 6302 */
6211 6303 ASSERT(sq->sq_evhead == bp);
6212 6304 sq->sq_evhead = bp->b_next;
6213 6305
6214 6306 if (bp->b_next == NULL) {
6215 6307 /* Deleting last */
6216 6308 ASSERT(sq->sq_evtail == bp);
6217 6309 sq->sq_evtail = NULL;
6218 6310 sq->sq_flags &= ~SQ_EVENTS;
6219 6311 }
6220 6312 bp->b_prev = bp->b_next = NULL;
6221 6313 ASSERT(bp->b_datap->db_ref != 0);
6222 6314
6223 6315 mutex_exit(SQLOCK(sq));
6224 6316
6225 6317 (*func)(qp, bp);
6226 6318
6227 6319 mutex_enter(SQLOCK(sq));
6228 6320 /*
6229 6321 * re-read the flags, since they could have changed.
6230 6322 */
6231 6323 flags = sq->sq_flags;
6232 6324 ASSERT(flags & SQ_EXCL);
6233 6325 }
6234 6326 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6235 6327 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6236 6328
6237 6329 if (flags & SQ_WANTWAKEUP) {
6238 6330 flags &= ~SQ_WANTWAKEUP;
6239 6331 cv_broadcast(&sq->sq_wait);
6240 6332 }
6241 6333 if (flags & SQ_WANTEXWAKEUP) {
6242 6334 flags &= ~SQ_WANTEXWAKEUP;
6243 6335 cv_broadcast(&sq->sq_exitwait);
6244 6336 }
6245 6337 sq->sq_flags = flags;
6246 6338 }
6247 6339
6248 6340 /*
6249 6341 * Put messages on the event list.
6250 6342 * If we can go exclusive now, do so and process the event list, otherwise
6251 6343 * let the last claim service this list (or wake the sqthread).
6252 6344 * This procedure assumes SQLOCK is held. To run the event list, it
6253 6345 * must be called with no claims.
6254 6346 */
6255 6347 static void
6256 6348 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6257 6349 {
6258 6350 uint16_t count;
6259 6351
6260 6352 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6261 6353 ASSERT(func != NULL);
6262 6354
6263 6355 /*
6264 6356 * This is a callback. Add it to the list of callbacks
6265 6357 * and see about upgrading.
6266 6358 */
6267 6359 mp->b_prev = (mblk_t *)func;
6268 6360 mp->b_queue = q;
6269 6361 mp->b_next = NULL;
6270 6362 if (sq->sq_evhead == NULL) {
6271 6363 sq->sq_evhead = sq->sq_evtail = mp;
6272 6364 sq->sq_flags |= SQ_EVENTS;
6273 6365 } else {
6274 6366 ASSERT(sq->sq_evtail != NULL);
6275 6367 ASSERT(sq->sq_evtail->b_next == NULL);
6276 6368 ASSERT(sq->sq_flags & SQ_EVENTS);
6277 6369 sq->sq_evtail->b_next = mp;
6278 6370 sq->sq_evtail = mp;
6279 6371 }
6280 6372 /*
6281 6373 * We have set SQ_EVENTS, so threads will have to
6282 6374 * unwind out of the perimeter, and new entries will
6283 6375 * not grab a putlock. But we still need to know
6284 6376 * how many threads have already made a claim to the
6285 6377 * syncq, so grab the putlocks, and sum the counts.
6286 6378 * If there are no claims on the syncq, we can upgrade
6287 6379 * to exclusive, and run the event list.
6288 6380 * NOTE: We hold the SQLOCK, so we can just grab the
6289 6381 * putlocks.
6290 6382 */
6291 6383 count = sq->sq_count;
6292 6384 SQ_PUTLOCKS_ENTER(sq);
6293 6385 SUM_SQ_PUTCOUNTS(sq, count);
6294 6386 /*
6295 6387 * We have no claim, so we need to check if there
6296 6388 * are no others, then we can upgrade.
6297 6389 */
6298 6390 /*
6299 6391 * There are currently no claims on
6300 6392 * the syncq by this thread (at least on this entry). The thread who has
6301 6393 * the claim should drain syncq.
6302 6394 */
6303 6395 if (count > 0) {
6304 6396 /*
6305 6397 * Can't upgrade - other threads inside.
6306 6398 */
6307 6399 SQ_PUTLOCKS_EXIT(sq);
6308 6400 mutex_exit(SQLOCK(sq));
6309 6401 return;
6310 6402 }
6311 6403 /*
6312 6404 * Need to set SQ_EXCL and make a claim on the syncq.
6313 6405 */
6314 6406 ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6315 6407 sq->sq_flags |= SQ_EXCL;
6316 6408 ASSERT(sq->sq_count == 0);
6317 6409 sq->sq_count++;
6318 6410 SQ_PUTLOCKS_EXIT(sq);
6319 6411
6320 6412 /* Process the events list */
6321 6413 sq_run_events(sq);
6322 6414
6323 6415 /*
6324 6416 * Release our claim...
6325 6417 */
6326 6418 sq->sq_count--;
6327 6419
6328 6420 /*
6329 6421 * And release SQ_EXCL.
6330 6422 * We don't need to acquire the putlocks to release
6331 6423 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6332 6424 */
6333 6425 sq->sq_flags &= ~SQ_EXCL;
6334 6426
6335 6427 /*
6336 6428 * sq_run_events should have released SQ_EXCL
6337 6429 */
6338 6430 ASSERT(!(sq->sq_flags & SQ_EXCL));
6339 6431
6340 6432 /*
6341 6433 * If anything happened while we were running the
6342 6434 * events (or was there before), we need to process
6343 6435 * them now. We shouldn't be exclusive sine we
6344 6436 * released the perimeter above (plus, we asserted
6345 6437 * for it).
6346 6438 */
6347 6439 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6348 6440 drain_syncq(sq);
6349 6441 else
6350 6442 mutex_exit(SQLOCK(sq));
6351 6443 }
6352 6444
6353 6445 /*
6354 6446 * Perform delayed processing. The caller has to make sure that it is safe
6355 6447 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6356 6448 * set).
6357 6449 *
6358 6450 * Assume that the caller has NO claims on the syncq. However, a claim
6359 6451 * on the syncq does not indicate that a thread is draining the syncq.
6360 6452 * There may be more claims on the syncq than there are threads draining
6361 6453 * (i.e. #_threads_draining <= sq_count)
6362 6454 *
6363 6455 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6364 6456 * in order to preserve qwriter(OUTER) ordering constraints.
6365 6457 *
6366 6458 * sq_putcount only needs to be checked when dispatching the queued
6367 6459 * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6368 6460 */
6369 6461 void
6370 6462 drain_syncq(syncq_t *sq)
6371 6463 {
6372 6464 queue_t *qp;
6373 6465 uint16_t count;
6374 6466 uint16_t type = sq->sq_type;
6375 6467 uint16_t flags = sq->sq_flags;
6376 6468 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE;
6377 6469
6378 6470 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6379 6471 "drain_syncq start:%p", sq);
6380 6472 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6381 6473 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6382 6474 sq->sq_oprev == NULL) ||
6383 6475 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6384 6476 sq->sq_oprev != NULL));
6385 6477
6386 6478 /*
6387 6479 * Drop SQ_SERVICE flag.
6388 6480 */
6389 6481 if (bg_service)
6390 6482 sq->sq_svcflags &= ~SQ_SERVICE;
6391 6483
6392 6484 /*
6393 6485 * If SQ_EXCL is set, someone else is processing this syncq - let him
6394 6486 * finish the job.
6395 6487 */
6396 6488 if (flags & SQ_EXCL) {
6397 6489 if (bg_service) {
6398 6490 ASSERT(sq->sq_servcount != 0);
6399 6491 sq->sq_servcount--;
6400 6492 }
6401 6493 mutex_exit(SQLOCK(sq));
6402 6494 return;
6403 6495 }
6404 6496
6405 6497 /*
6406 6498 * This routine can be called by a background thread if
6407 6499 * it was scheduled by a hi-priority thread. SO, if there are
6408 6500 * NOT messages queued, return (remember, we have the SQLOCK,
6409 6501 * and it cannot change until we release it). Wakeup any waiters also.
6410 6502 */
6411 6503 if (!(flags & SQ_QUEUED)) {
6412 6504 if (flags & SQ_WANTWAKEUP) {
6413 6505 flags &= ~SQ_WANTWAKEUP;
6414 6506 cv_broadcast(&sq->sq_wait);
6415 6507 }
6416 6508 if (flags & SQ_WANTEXWAKEUP) {
6417 6509 flags &= ~SQ_WANTEXWAKEUP;
6418 6510 cv_broadcast(&sq->sq_exitwait);
6419 6511 }
6420 6512 sq->sq_flags = flags;
6421 6513 if (bg_service) {
6422 6514 ASSERT(sq->sq_servcount != 0);
6423 6515 sq->sq_servcount--;
6424 6516 }
6425 6517 mutex_exit(SQLOCK(sq));
6426 6518 return;
6427 6519 }
6428 6520
6429 6521 /*
6430 6522 * If this is not a concurrent put perimeter, we need to
6431 6523 * become exclusive to drain. Also, if not CIPUT, we would
6432 6524 * not have acquired a putlock, so we don't need to check
6433 6525 * the putcounts. If not entering with a claim, we test
6434 6526 * for sq_count == 0.
6435 6527 */
6436 6528 type = sq->sq_type;
6437 6529 if (!(type & SQ_CIPUT)) {
6438 6530 if (sq->sq_count > 1) {
6439 6531 if (bg_service) {
6440 6532 ASSERT(sq->sq_servcount != 0);
6441 6533 sq->sq_servcount--;
6442 6534 }
6443 6535 mutex_exit(SQLOCK(sq));
6444 6536 return;
6445 6537 }
6446 6538 sq->sq_flags |= SQ_EXCL;
6447 6539 }
6448 6540
6449 6541 /*
6450 6542 * This is where we make a claim to the syncq.
6451 6543 * This can either be done by incrementing a putlock, or
6452 6544 * the sq_count. But since we already have the SQLOCK
6453 6545 * here, we just bump the sq_count.
6454 6546 *
6455 6547 * Note that after we make a claim, we need to let the code
6456 6548 * fall through to the end of this routine to clean itself
6457 6549 * up. A return in the while loop will put the syncq in a
6458 6550 * very bad state.
6459 6551 */
6460 6552 sq->sq_count++;
6461 6553 ASSERT(sq->sq_count != 0); /* wraparound */
6462 6554
6463 6555 while ((flags = sq->sq_flags) & SQ_QUEUED) {
6464 6556 /*
6465 6557 * If we are told to stayaway or went exclusive,
6466 6558 * we are done.
6467 6559 */
6468 6560 if (flags & (SQ_STAYAWAY)) {
6469 6561 break;
6470 6562 }
6471 6563
6472 6564 /*
6473 6565 * If there are events to run, do so.
6474 6566 * We have one claim to the syncq, so if there are
6475 6567 * more than one, other threads are running.
6476 6568 */
6477 6569 if (sq->sq_evhead != NULL) {
6478 6570 ASSERT(sq->sq_flags & SQ_EVENTS);
6479 6571
6480 6572 count = sq->sq_count;
6481 6573 SQ_PUTLOCKS_ENTER(sq);
6482 6574 SUM_SQ_PUTCOUNTS(sq, count);
6483 6575 if (count > 1) {
6484 6576 SQ_PUTLOCKS_EXIT(sq);
6485 6577 /* Can't upgrade - other threads inside */
6486 6578 break;
6487 6579 }
6488 6580 ASSERT((flags & SQ_EXCL) == 0);
6489 6581 sq->sq_flags = flags | SQ_EXCL;
6490 6582 SQ_PUTLOCKS_EXIT(sq);
6491 6583 /*
6492 6584 * we have the only claim, run the events,
6493 6585 * sq_run_events will clear the SQ_EXCL flag.
6494 6586 */
6495 6587 sq_run_events(sq);
6496 6588
6497 6589 /*
6498 6590 * If this is a CIPUT perimeter, we need
6499 6591 * to drop the SQ_EXCL flag so we can properly
6500 6592 * continue draining the syncq.
6501 6593 */
6502 6594 if (type & SQ_CIPUT) {
6503 6595 ASSERT(sq->sq_flags & SQ_EXCL);
6504 6596 sq->sq_flags &= ~SQ_EXCL;
6505 6597 }
6506 6598
6507 6599 /*
6508 6600 * And go back to the beginning just in case
6509 6601 * anything changed while we were away.
6510 6602 */
6511 6603 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6512 6604 continue;
6513 6605 }
6514 6606
6515 6607 ASSERT(sq->sq_evhead == NULL);
6516 6608 ASSERT(!(sq->sq_flags & SQ_EVENTS));
6517 6609
6518 6610 /*
6519 6611 * Find the queue that is not draining.
6520 6612 *
6521 6613 * q_draining is protected by QLOCK which we do not hold.
6522 6614 * But if it was set, then a thread was draining, and if it gets
6523 6615 * cleared, then it was because the thread has successfully
6524 6616 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6525 6617 * state to happen, a thread needs the SQLOCK which we hold, and
6526 6618 * if there was such a flag, we would have already seen it.
6527 6619 */
6528 6620
6529 6621 for (qp = sq->sq_head;
6530 6622 qp != NULL && (qp->q_draining ||
6531 6623 (qp->q_sqflags & Q_SQDRAINING));
6532 6624 qp = qp->q_sqnext)
6533 6625 ;
6534 6626
6535 6627 if (qp == NULL)
6536 6628 break;
6537 6629
6538 6630 /*
6539 6631 * We have a queue to work on, and we hold the
6540 6632 * SQLOCK and one claim, call qdrain_syncq.
6541 6633 * This means we need to release the SQLOCK and
6542 6634 * acquire the QLOCK (OK since we have a claim).
6543 6635 * Note that qdrain_syncq will actually dequeue
6544 6636 * this queue from the sq_head list when it is
6545 6637 * convinced all the work is done and release
6546 6638 * the QLOCK before returning.
6547 6639 */
6548 6640 qp->q_sqflags |= Q_SQDRAINING;
6549 6641 mutex_exit(SQLOCK(sq));
6550 6642 mutex_enter(QLOCK(qp));
6551 6643 qdrain_syncq(sq, qp);
6552 6644 mutex_enter(SQLOCK(sq));
6553 6645
6554 6646 /* The queue is drained */
6555 6647 ASSERT(qp->q_sqflags & Q_SQDRAINING);
6556 6648 qp->q_sqflags &= ~Q_SQDRAINING;
6557 6649 /*
6558 6650 * NOTE: After this point qp should not be used since it may be
6559 6651 * closed.
6560 6652 */
6561 6653 }
6562 6654
6563 6655 ASSERT(MUTEX_HELD(SQLOCK(sq)));
6564 6656 flags = sq->sq_flags;
6565 6657
6566 6658 /*
6567 6659 * sq->sq_head cannot change because we hold the
6568 6660 * sqlock. However, a thread CAN decide that it is no longer
6569 6661 * going to drain that queue. However, this should be due to
6570 6662 * a GOAWAY state, and we should see that here.
6571 6663 *
6572 6664 * This loop is not very efficient. One solution may be adding a second
6573 6665 * pointer to the "draining" queue, but it is difficult to do when
6574 6666 * queues are inserted in the middle due to priority ordering. Another
6575 6667 * possibility is to yank the queue out of the sq list and put it onto
6576 6668 * the "draining list" and then put it back if it can't be drained.
6577 6669 */
6578 6670
6579 6671 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6580 6672 (type & SQ_CI) || sq->sq_head->q_draining);
6581 6673
6582 6674 /* Drop SQ_EXCL for non-CIPUT perimeters */
6583 6675 if (!(type & SQ_CIPUT))
6584 6676 flags &= ~SQ_EXCL;
6585 6677 ASSERT((flags & SQ_EXCL) == 0);
6586 6678
6587 6679 /* Wake up any waiters. */
6588 6680 if (flags & SQ_WANTWAKEUP) {
6589 6681 flags &= ~SQ_WANTWAKEUP;
6590 6682 cv_broadcast(&sq->sq_wait);
6591 6683 }
6592 6684 if (flags & SQ_WANTEXWAKEUP) {
6593 6685 flags &= ~SQ_WANTEXWAKEUP;
6594 6686 cv_broadcast(&sq->sq_exitwait);
6595 6687 }
6596 6688 sq->sq_flags = flags;
6597 6689
6598 6690 ASSERT(sq->sq_count != 0);
6599 6691 /* Release our claim. */
6600 6692 sq->sq_count--;
6601 6693
6602 6694 if (bg_service) {
6603 6695 ASSERT(sq->sq_servcount != 0);
6604 6696 sq->sq_servcount--;
6605 6697 }
6606 6698
6607 6699 mutex_exit(SQLOCK(sq));
6608 6700
6609 6701 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6610 6702 "drain_syncq end:%p", sq);
6611 6703 }
6612 6704
6613 6705
6614 6706 /*
6615 6707 *
6616 6708 * qdrain_syncq can be called (currently) from only one of two places:
6617 6709 * drain_syncq
6618 6710 * putnext (or some variation of it).
6619 6711 * and eventually
6620 6712 * qwait(_sig)
6621 6713 *
6622 6714 * If called from drain_syncq, we found it in the list of queues needing
6623 6715 * service, so there is work to be done (or it wouldn't be in the list).
6624 6716 *
6625 6717 * If called from some putnext variation, it was because the
6626 6718 * perimeter is open, but messages are blocking a putnext and
6627 6719 * there is not a thread working on it. Now a thread could start
6628 6720 * working on it while we are getting ready to do so ourself, but
6629 6721 * the thread would set the q_draining flag, and we can spin out.
6630 6722 *
6631 6723 * As for qwait(_sig), I think I shall let it continue to call
6632 6724 * drain_syncq directly (after all, it will get here eventually).
6633 6725 *
6634 6726 * qdrain_syncq has to terminate when:
6635 6727 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6636 6728 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6637 6729 *
6638 6730 * ASSUMES:
6639 6731 * One claim
6640 6732 * QLOCK held
6641 6733 * SQLOCK not held
6642 6734 * Will release QLOCK before returning
6643 6735 */
6644 6736 void
6645 6737 qdrain_syncq(syncq_t *sq, queue_t *q)
6646 6738 {
6647 6739 mblk_t *bp;
6648 6740 #ifdef DEBUG
6649 6741 uint16_t count;
6650 6742 #endif
6651 6743
6652 6744 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6653 6745 "drain_syncq start:%p", sq);
6654 6746 ASSERT(q->q_syncq == sq);
6655 6747 ASSERT(MUTEX_HELD(QLOCK(q)));
6656 6748 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6657 6749 /*
6658 6750 * For non-CIPUT perimeters, we should be called with the exclusive bit
6659 6751 * set already. For CIPUT perimeters, we will be doing a concurrent
6660 6752 * drain, so it better not be set.
6661 6753 */
6662 6754 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6663 6755 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6664 6756 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6665 6757 /*
6666 6758 * All outer pointers are set, or none of them are
6667 6759 */
6668 6760 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6669 6761 sq->sq_oprev == NULL) ||
6670 6762 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6671 6763 sq->sq_oprev != NULL));
6672 6764 #ifdef DEBUG
6673 6765 count = sq->sq_count;
6674 6766 /*
6675 6767 * This is OK without the putlocks, because we have one
6676 6768 * claim either from the sq_count, or a putcount. We could
6677 6769 * get an erroneous value from other counts, but ours won't
6678 6770 * change, so one way or another, we will have at least a
6679 6771 * value of one.
6680 6772 */
6681 6773 SUM_SQ_PUTCOUNTS(sq, count);
6682 6774 ASSERT(count >= 1);
6683 6775 #endif /* DEBUG */
6684 6776
6685 6777 /*
6686 6778 * The first thing to do is find out if a thread is already draining
6687 6779 * this queue. If so, we are done, just return.
6688 6780 */
6689 6781 if (q->q_draining) {
6690 6782 mutex_exit(QLOCK(q));
6691 6783 return;
6692 6784 }
6693 6785
6694 6786 /*
6695 6787 * If the perimeter is exclusive, there is nothing we can do right now,
6696 6788 * go away. Note that there is nothing to prevent this case from
6697 6789 * changing right after this check, but the spin-out will catch it.
6698 6790 */
6699 6791
6700 6792 /* Tell other threads that we are draining this queue */
6701 6793 q->q_draining = 1; /* Protected by QLOCK */
6702 6794
6703 6795 /*
6704 6796 * If there is nothing to do, clear QFULL as necessary. This caters for
6705 6797 * the case where an empty queue was enqueued onto the syncq.
6706 6798 */
6707 6799 if (q->q_sqhead == NULL) {
6708 6800 ASSERT(q->q_syncqmsgs == 0);
6709 6801 mutex_exit(QLOCK(q));
6710 6802 clr_qfull(q);
6711 6803 mutex_enter(QLOCK(q));
6712 6804 }
6713 6805
6714 6806 /*
6715 6807 * Note that q_sqhead must be re-checked here in case another message
6716 6808 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6717 6809 */
6718 6810 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6719 6811 /*
6720 6812 * Because we can enter this routine just because a putnext is
6721 6813 * blocked, we need to spin out if the perimeter wants to go
6722 6814 * exclusive as well as just blocked. We need to spin out also
6723 6815 * if events are queued on the syncq.
6724 6816 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6725 6817 * set it, and it can't become exclusive while we hold a claim.
6726 6818 */
6727 6819 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6728 6820 break;
6729 6821 }
6730 6822
6731 6823 #ifdef DEBUG
6732 6824 /*
6733 6825 * Since we are in qdrain_syncq, we already know the queue,
6734 6826 * but for sanity, we want to check this against the qp that
6735 6827 * was passed in by bp->b_queue.
6736 6828 */
6737 6829
6738 6830 ASSERT(bp->b_queue == q);
6739 6831 ASSERT(bp->b_queue->q_syncq == sq);
6740 6832 bp->b_queue = NULL;
6741 6833
6742 6834 /*
6743 6835 * We would have the following check in the DEBUG code:
6744 6836 *
6745 6837 * if (bp->b_prev != NULL) {
6746 6838 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6747 6839 * }
6748 6840 *
6749 6841 * This can't be done, however, since IP modifies qinfo
6750 6842 * structure at run-time (switching between IPv4 qinfo and IPv6
6751 6843 * qinfo), invalidating the check.
6752 6844 * So the assignment to func is left here, but the ASSERT itself
6753 6845 * is removed until the whole issue is resolved.
6754 6846 */
6755 6847 #endif
6756 6848 ASSERT(q->q_sqhead == bp);
6757 6849 q->q_sqhead = bp->b_next;
6758 6850 bp->b_prev = bp->b_next = NULL;
6759 6851 ASSERT(q->q_syncqmsgs > 0);
6760 6852 mutex_exit(QLOCK(q));
6761 6853
6762 6854 ASSERT(bp->b_datap->db_ref != 0);
6763 6855
6764 6856 (void) (*q->q_qinfo->qi_putp)(q, bp);
6765 6857
6766 6858 mutex_enter(QLOCK(q));
6767 6859
6768 6860 /*
6769 6861 * q_syncqmsgs should only be decremented after executing the
6770 6862 * put procedure to avoid message re-ordering. This is due to an
6771 6863 * optimisation in putnext() which can call the put procedure
6772 6864 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6773 6865 * being set).
6774 6866 *
6775 6867 * We also need to clear QFULL in the next service procedure
6776 6868 * queue if this is the last message destined for that queue.
6777 6869 *
6778 6870 * It would make better sense to have some sort of tunable for
6779 6871 * the low water mark, but these semantics are not yet defined.
6780 6872 * So, alas, we use a constant.
6781 6873 */
6782 6874 if (--q->q_syncqmsgs == 0) {
6783 6875 mutex_exit(QLOCK(q));
6784 6876 clr_qfull(q);
6785 6877 mutex_enter(QLOCK(q));
6786 6878 }
6787 6879
6788 6880 /*
6789 6881 * Always clear SQ_EXCL when CIPUT in order to handle
6790 6882 * qwriter(INNER). The putp() can call qwriter and get exclusive
6791 6883 * access IFF this is the only claim. So, we need to test for
6792 6884 * this possibility, acquire the mutex and clear the bit.
6793 6885 */
6794 6886 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6795 6887 mutex_enter(SQLOCK(sq));
6796 6888 sq->sq_flags &= ~SQ_EXCL;
6797 6889 mutex_exit(SQLOCK(sq));
6798 6890 }
6799 6891 }
6800 6892
6801 6893 /*
6802 6894 * We should either have no messages on this queue, or we were told to
6803 6895 * goaway by a waiter (which we will wake up at the end of this
6804 6896 * function).
6805 6897 */
6806 6898 ASSERT((q->q_sqhead == NULL) ||
6807 6899 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6808 6900
6809 6901 ASSERT(MUTEX_HELD(QLOCK(q)));
6810 6902 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6811 6903
6812 6904 /* Remove the q from the syncq list if all the messages are drained. */
6813 6905 if (q->q_sqhead == NULL) {
6814 6906 ASSERT(q->q_syncqmsgs == 0);
6815 6907 mutex_enter(SQLOCK(sq));
6816 6908 if (q->q_sqflags & Q_SQQUEUED)
6817 6909 SQRM_Q(sq, q);
6818 6910 mutex_exit(SQLOCK(sq));
6819 6911 /*
6820 6912 * Since the queue is removed from the list, reset its priority.
6821 6913 */
6822 6914 q->q_spri = 0;
6823 6915 }
6824 6916
6825 6917 /*
6826 6918 * Remember, the q_draining flag is used to let another thread know
6827 6919 * that there is a thread currently draining the messages for a queue.
6828 6920 * Since we are now done with this queue (even if there may be messages
6829 6921 * still there), we need to clear this flag so some thread will work on
6830 6922 * it if needed.
6831 6923 */
6832 6924 ASSERT(q->q_draining);
6833 6925 q->q_draining = 0;
6834 6926
6835 6927 /* Called with a claim, so OK to drop all locks. */
6836 6928 mutex_exit(QLOCK(q));
6837 6929
6838 6930 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6839 6931 "drain_syncq end:%p", sq);
6840 6932 }
6841 6933 /* END OF QDRAIN_SYNCQ */
6842 6934
6843 6935
6844 6936 /*
6845 6937 * This is the mate to qdrain_syncq, except that it is putting the message onto
6846 6938 * the queue instead of draining. Since the message is destined for the queue
6847 6939 * that is selected, there is no need to identify the function because the
6848 6940 * message is intended for the put routine for the queue. For debug kernels,
6849 6941 * this routine will do it anyway just in case.
6850 6942 *
6851 6943 * After the message is enqueued on the syncq, it calls putnext_tail()
6852 6944 * which will schedule a background thread to actually process the message.
6853 6945 *
6854 6946 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6855 6947 * SQLOCK(sq) and QLOCK(q) are not held.
6856 6948 */
6857 6949 void
6858 6950 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6859 6951 {
6860 6952 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6861 6953 ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6862 6954 ASSERT(sq->sq_count > 0);
6863 6955 ASSERT(q->q_syncq == sq);
6864 6956 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6865 6957 sq->sq_oprev == NULL) ||
6866 6958 (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6867 6959 sq->sq_oprev != NULL));
6868 6960
6869 6961 mutex_enter(QLOCK(q));
6870 6962
6871 6963 #ifdef DEBUG
6872 6964 /*
6873 6965 * This is used for debug in the qfill_syncq/qdrain_syncq case
6874 6966 * to trace the queue that the message is intended for. Note
6875 6967 * that the original use was to identify the queue and function
6876 6968 * to call on the drain. In the new syncq, we have the context
6877 6969 * of the queue that we are draining, so call it's putproc and
6878 6970 * don't rely on the saved values. But for debug this is still
6879 6971 * useful information.
6880 6972 */
6881 6973 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6882 6974 mp->b_queue = q;
6883 6975 mp->b_next = NULL;
6884 6976 #endif
6885 6977 ASSERT(q->q_syncq == sq);
6886 6978 /*
6887 6979 * Enqueue the message on the list.
6888 6980 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to
6889 6981 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP().
6890 6982 */
6891 6983 SQPUT_MP(q, mp);
6892 6984 mutex_enter(SQLOCK(sq));
6893 6985
6894 6986 /*
6895 6987 * And queue on syncq for scheduling, if not already queued.
6896 6988 * Note that we need the SQLOCK for this, and for testing flags
6897 6989 * at the end to see if we will drain. So grab it now, and
6898 6990 * release it before we call qdrain_syncq or return.
6899 6991 */
6900 6992 if (!(q->q_sqflags & Q_SQQUEUED)) {
6901 6993 q->q_spri = curthread->t_pri;
6902 6994 SQPUT_Q(sq, q);
6903 6995 }
6904 6996 #ifdef DEBUG
6905 6997 else {
6906 6998 /*
6907 6999 * All of these conditions MUST be true!
6908 7000 */
6909 7001 ASSERT(sq->sq_tail != NULL);
6910 7002 if (sq->sq_tail == sq->sq_head) {
6911 7003 ASSERT((q->q_sqprev == NULL) &&
6912 7004 (q->q_sqnext == NULL));
6913 7005 } else {
6914 7006 ASSERT((q->q_sqprev != NULL) ||
6915 7007 (q->q_sqnext != NULL));
6916 7008 }
6917 7009 ASSERT(sq->sq_flags & SQ_QUEUED);
6918 7010 ASSERT(q->q_syncqmsgs != 0);
6919 7011 ASSERT(q->q_sqflags & Q_SQQUEUED);
6920 7012 }
6921 7013 #endif
6922 7014 mutex_exit(QLOCK(q));
6923 7015 /*
6924 7016 * SQLOCK is still held, so sq_count can be safely decremented.
6925 7017 */
6926 7018 sq->sq_count--;
6927 7019
6928 7020 putnext_tail(sq, q, 0);
6929 7021 /* Should not reference sq or q after this point. */
6930 7022 }
6931 7023
6932 7024 /* End of qfill_syncq */
6933 7025
6934 7026 /*
6935 7027 * Remove all messages from a syncq (if qp is NULL) or remove all messages
6936 7028 * that would be put into qp by drain_syncq.
6937 7029 * Used when deleting the syncq (qp == NULL) or when detaching
6938 7030 * a queue (qp != NULL).
6939 7031 * Return non-zero if one or more messages were freed.
6940 7032 *
6941 7033 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6942 7034 * sq_putlocks are used.
6943 7035 *
6944 7036 * NOTE: This function assumes that it is called from the close() context and
6945 7037 * that all the queues in the syncq are going away. For this reason it doesn't
6946 7038 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6947 7039 * currently valid, but it is useful to rethink this function to behave properly
6948 7040 * in other cases.
6949 7041 */
6950 7042 int
6951 7043 flush_syncq(syncq_t *sq, queue_t *qp)
6952 7044 {
6953 7045 mblk_t *bp, *mp_head, *mp_next, *mp_prev;
6954 7046 queue_t *q;
6955 7047 int ret = 0;
6956 7048
6957 7049 mutex_enter(SQLOCK(sq));
6958 7050
6959 7051 /*
6960 7052 * Before we leave, we need to make sure there are no
6961 7053 * events listed for this queue. All events for this queue
6962 7054 * will just be freed.
6963 7055 */
6964 7056 if (qp != NULL && sq->sq_evhead != NULL) {
6965 7057 ASSERT(sq->sq_flags & SQ_EVENTS);
6966 7058
6967 7059 mp_prev = NULL;
6968 7060 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6969 7061 mp_next = bp->b_next;
6970 7062 if (bp->b_queue == qp) {
6971 7063 /* Delete this message */
6972 7064 if (mp_prev != NULL) {
6973 7065 mp_prev->b_next = mp_next;
6974 7066 /*
6975 7067 * Update sq_evtail if the last element
6976 7068 * is removed.
6977 7069 */
6978 7070 if (bp == sq->sq_evtail) {
6979 7071 ASSERT(mp_next == NULL);
6980 7072 sq->sq_evtail = mp_prev;
6981 7073 }
6982 7074 } else
6983 7075 sq->sq_evhead = mp_next;
6984 7076 if (sq->sq_evhead == NULL)
6985 7077 sq->sq_flags &= ~SQ_EVENTS;
6986 7078 bp->b_prev = bp->b_next = NULL;
6987 7079 freemsg(bp);
6988 7080 ret++;
6989 7081 } else {
6990 7082 mp_prev = bp;
6991 7083 }
6992 7084 }
6993 7085 }
6994 7086
6995 7087 /*
6996 7088 * Walk sq_head and:
6997 7089 * - match qp if qp is set, remove it's messages
6998 7090 * - all if qp is not set
6999 7091 */
7000 7092 q = sq->sq_head;
7001 7093 while (q != NULL) {
7002 7094 ASSERT(q->q_syncq == sq);
7003 7095 if ((qp == NULL) || (qp == q)) {
7004 7096 /*
7005 7097 * Yank the messages as a list off the queue
7006 7098 */
7007 7099 mp_head = q->q_sqhead;
7008 7100 /*
7009 7101 * We do not have QLOCK(q) here (which is safe due to
7010 7102 * assumptions mentioned above). To obtain the lock we
7011 7103 * need to release SQLOCK which may allow lots of things
7012 7104 * to change upon us. This place requires more analysis.
7013 7105 */
7014 7106 q->q_sqhead = q->q_sqtail = NULL;
7015 7107 ASSERT(mp_head->b_queue &&
7016 7108 mp_head->b_queue->q_syncq == sq);
7017 7109
7018 7110 /*
7019 7111 * Free each of the messages.
7020 7112 */
7021 7113 for (bp = mp_head; bp != NULL; bp = mp_next) {
7022 7114 mp_next = bp->b_next;
7023 7115 bp->b_prev = bp->b_next = NULL;
7024 7116 freemsg(bp);
7025 7117 ret++;
7026 7118 }
7027 7119 /*
7028 7120 * Now remove the queue from the syncq.
7029 7121 */
7030 7122 ASSERT(q->q_sqflags & Q_SQQUEUED);
7031 7123 SQRM_Q(sq, q);
7032 7124 q->q_spri = 0;
7033 7125 q->q_syncqmsgs = 0;
7034 7126
7035 7127 /*
7036 7128 * If qp was specified, we are done with it and are
7037 7129 * going to drop SQLOCK(sq) and return. We wakeup syncq
7038 7130 * waiters while we still have the SQLOCK.
7039 7131 */
7040 7132 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7041 7133 sq->sq_flags &= ~SQ_WANTWAKEUP;
7042 7134 cv_broadcast(&sq->sq_wait);
7043 7135 }
7044 7136 /* Drop SQLOCK across clr_qfull */
7045 7137 mutex_exit(SQLOCK(sq));
7046 7138
7047 7139 /*
7048 7140 * We avoid doing the test that drain_syncq does and
7049 7141 * unconditionally clear qfull for every flushed
7050 7142 * message. Since flush_syncq is only called during
7051 7143 * close this should not be a problem.
7052 7144 */
7053 7145 clr_qfull(q);
7054 7146 if (qp != NULL) {
7055 7147 return (ret);
7056 7148 } else {
7057 7149 mutex_enter(SQLOCK(sq));
7058 7150 /*
7059 7151 * The head was removed by SQRM_Q above.
7060 7152 * reread the new head and flush it.
7061 7153 */
7062 7154 q = sq->sq_head;
7063 7155 }
7064 7156 } else {
7065 7157 q = q->q_sqnext;
7066 7158 }
7067 7159 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7068 7160 }
7069 7161
7070 7162 if (sq->sq_flags & SQ_WANTWAKEUP) {
7071 7163 sq->sq_flags &= ~SQ_WANTWAKEUP;
7072 7164 cv_broadcast(&sq->sq_wait);
7073 7165 }
7074 7166
7075 7167 mutex_exit(SQLOCK(sq));
7076 7168 return (ret);
7077 7169 }
7078 7170
7079 7171 /*
7080 7172 * Propagate all messages from a syncq to the next syncq that are associated
7081 7173 * with the specified queue. If the queue is attached to a driver or if the
7082 7174 * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7083 7175 *
7084 7176 * Assumes that the stream is strlock()'ed. We don't come here if there
7085 7177 * are no messages to propagate.
7086 7178 *
7087 7179 * NOTE : If the queue is attached to a driver, all the messages are freed
7088 7180 * as there is no point in propagating the messages from the driver syncq
7089 7181 * to the closing stream head which will in turn get freed later.
7090 7182 */
7091 7183 static int
7092 7184 propagate_syncq(queue_t *qp)
7093 7185 {
7094 7186 mblk_t *bp, *head, *tail, *prev, *next;
7095 7187 syncq_t *sq;
7096 7188 queue_t *nqp;
7097 7189 syncq_t *nsq;
7098 7190 boolean_t isdriver;
7099 7191 int moved = 0;
7100 7192 uint16_t flags;
7101 7193 pri_t priority = curthread->t_pri;
7102 7194 #ifdef DEBUG
7103 7195 void (*func)();
7104 7196 #endif
7105 7197
7106 7198 sq = qp->q_syncq;
7107 7199 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7108 7200 /* debug macro */
7109 7201 SQ_PUTLOCKS_HELD(sq);
7110 7202 /*
7111 7203 * As entersq() does not increment the sq_count for
7112 7204 * the write side, check sq_count for non-QPERQ
7113 7205 * perimeters alone.
7114 7206 */
7115 7207 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7116 7208
7117 7209 /*
7118 7210 * propagate_syncq() can be called because of either messages on the
7119 7211 * queue syncq or because on events on the queue syncq. Do actual
7120 7212 * message propagations if there are any messages.
7121 7213 */
7122 7214 if (qp->q_syncqmsgs) {
7123 7215 isdriver = (qp->q_flag & QISDRV);
7124 7216
7125 7217 if (!isdriver) {
7126 7218 nqp = qp->q_next;
7127 7219 nsq = nqp->q_syncq;
7128 7220 ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7129 7221 /* debug macro */
7130 7222 SQ_PUTLOCKS_HELD(nsq);
7131 7223 #ifdef DEBUG
7132 7224 func = (void (*)())nqp->q_qinfo->qi_putp;
7133 7225 #endif
7134 7226 }
7135 7227
7136 7228 SQRM_Q(sq, qp);
7137 7229 priority = MAX(qp->q_spri, priority);
7138 7230 qp->q_spri = 0;
7139 7231 head = qp->q_sqhead;
7140 7232 tail = qp->q_sqtail;
7141 7233 qp->q_sqhead = qp->q_sqtail = NULL;
7142 7234 qp->q_syncqmsgs = 0;
7143 7235
7144 7236 /*
7145 7237 * Walk the list of messages, and free them if this is a driver,
7146 7238 * otherwise reset the b_prev and b_queue value to the new putp.
7147 7239 * Afterward, we will just add the head to the end of the next
7148 7240 * syncq, and point the tail to the end of this one.
7149 7241 */
7150 7242
7151 7243 for (bp = head; bp != NULL; bp = next) {
7152 7244 next = bp->b_next;
7153 7245 if (isdriver) {
7154 7246 bp->b_prev = bp->b_next = NULL;
7155 7247 freemsg(bp);
7156 7248 continue;
7157 7249 }
7158 7250 /* Change the q values for this message */
7159 7251 bp->b_queue = nqp;
7160 7252 #ifdef DEBUG
7161 7253 bp->b_prev = (mblk_t *)func;
7162 7254 #endif
7163 7255 moved++;
7164 7256 }
7165 7257 /*
7166 7258 * Attach list of messages to the end of the new queue (if there
7167 7259 * is a list of messages).
7168 7260 */
7169 7261
7170 7262 if (!isdriver && head != NULL) {
7171 7263 ASSERT(tail != NULL);
7172 7264 if (nqp->q_sqhead == NULL) {
7173 7265 nqp->q_sqhead = head;
7174 7266 } else {
7175 7267 ASSERT(nqp->q_sqtail != NULL);
7176 7268 nqp->q_sqtail->b_next = head;
7177 7269 }
7178 7270 nqp->q_sqtail = tail;
7179 7271 /*
7180 7272 * When messages are moved from high priority queue to
7181 7273 * another queue, the destination queue priority is
7182 7274 * upgraded.
7183 7275 */
7184 7276
7185 7277 if (priority > nqp->q_spri)
7186 7278 nqp->q_spri = priority;
7187 7279
7188 7280 SQPUT_Q(nsq, nqp);
7189 7281
7190 7282 nqp->q_syncqmsgs += moved;
7191 7283 ASSERT(nqp->q_syncqmsgs != 0);
7192 7284 }
7193 7285 }
7194 7286
7195 7287 /*
7196 7288 * Before we leave, we need to make sure there are no
7197 7289 * events listed for this queue. All events for this queue
7198 7290 * will just be freed.
7199 7291 */
7200 7292 if (sq->sq_evhead != NULL) {
7201 7293 ASSERT(sq->sq_flags & SQ_EVENTS);
7202 7294 prev = NULL;
7203 7295 for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7204 7296 next = bp->b_next;
7205 7297 if (bp->b_queue == qp) {
7206 7298 /* Delete this message */
7207 7299 if (prev != NULL) {
7208 7300 prev->b_next = next;
7209 7301 /*
7210 7302 * Update sq_evtail if the last element
7211 7303 * is removed.
7212 7304 */
7213 7305 if (bp == sq->sq_evtail) {
7214 7306 ASSERT(next == NULL);
7215 7307 sq->sq_evtail = prev;
7216 7308 }
7217 7309 } else
7218 7310 sq->sq_evhead = next;
7219 7311 if (sq->sq_evhead == NULL)
7220 7312 sq->sq_flags &= ~SQ_EVENTS;
7221 7313 bp->b_prev = bp->b_next = NULL;
7222 7314 freemsg(bp);
7223 7315 } else {
7224 7316 prev = bp;
7225 7317 }
7226 7318 }
7227 7319 }
7228 7320
7229 7321 flags = sq->sq_flags;
7230 7322
7231 7323 /* Wake up any waiter before leaving. */
7232 7324 if (flags & SQ_WANTWAKEUP) {
7233 7325 flags &= ~SQ_WANTWAKEUP;
7234 7326 cv_broadcast(&sq->sq_wait);
7235 7327 }
7236 7328 sq->sq_flags = flags;
7237 7329
7238 7330 return (moved);
7239 7331 }
7240 7332
7241 7333 /*
7242 7334 * Try and upgrade to exclusive access at the inner perimeter. If this can
7243 7335 * not be done without blocking then request will be queued on the syncq
7244 7336 * and drain_syncq will run it later.
7245 7337 *
7246 7338 * This routine can only be called from put or service procedures plus
7247 7339 * asynchronous callback routines that have properly entered the queue (with
7248 7340 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7249 7341 * associated with q.
7250 7342 */
7251 7343 void
7252 7344 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7253 7345 {
7254 7346 syncq_t *sq = q->q_syncq;
7255 7347 uint16_t count;
7256 7348
7257 7349 mutex_enter(SQLOCK(sq));
7258 7350 count = sq->sq_count;
7259 7351 SQ_PUTLOCKS_ENTER(sq);
7260 7352 SUM_SQ_PUTCOUNTS(sq, count);
7261 7353 ASSERT(count >= 1);
7262 7354 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7263 7355
7264 7356 if (count == 1) {
7265 7357 /*
7266 7358 * Can upgrade. This case also handles nested qwriter calls
7267 7359 * (when the qwriter callback function calls qwriter). In that
7268 7360 * case SQ_EXCL is already set.
7269 7361 */
7270 7362 sq->sq_flags |= SQ_EXCL;
7271 7363 SQ_PUTLOCKS_EXIT(sq);
7272 7364 mutex_exit(SQLOCK(sq));
7273 7365 (*func)(q, mp);
7274 7366 /*
7275 7367 * Assumes that leavesq, putnext, and drain_syncq will reset
7276 7368 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7277 7369 * until putnext, leavesq, or drain_syncq drops it.
7278 7370 * That way we handle nested qwriter(INNER) without dropping
7279 7371 * SQ_EXCL until the outermost qwriter callback routine is
7280 7372 * done.
7281 7373 */
7282 7374 return;
7283 7375 }
7284 7376 SQ_PUTLOCKS_EXIT(sq);
7285 7377 sqfill_events(sq, q, mp, func);
7286 7378 }
7287 7379
7288 7380 /*
7289 7381 * Synchronous callback support functions
7290 7382 */
7291 7383
7292 7384 /*
7293 7385 * Allocate a callback parameter structure.
7294 7386 * Assumes that caller initializes the flags and the id.
7295 7387 * Acquires SQLOCK(sq) if non-NULL is returned.
7296 7388 */
7297 7389 callbparams_t *
7298 7390 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7299 7391 {
7300 7392 callbparams_t *cbp;
7301 7393 size_t size = sizeof (callbparams_t);
7302 7394
7303 7395 cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7304 7396
7305 7397 /*
7306 7398 * Only try tryhard allocation if the caller is ready to panic.
7307 7399 * Otherwise just fail.
7308 7400 */
7309 7401 if (cbp == NULL) {
7310 7402 if (kmflags & KM_PANIC)
7311 7403 cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7312 7404 &size, kmflags);
7313 7405 else
7314 7406 return (NULL);
7315 7407 }
7316 7408
7317 7409 ASSERT(size >= sizeof (callbparams_t));
7318 7410 cbp->cbp_size = size;
7319 7411 cbp->cbp_sq = sq;
7320 7412 cbp->cbp_func = func;
7321 7413 cbp->cbp_arg = arg;
7322 7414 mutex_enter(SQLOCK(sq));
7323 7415 cbp->cbp_next = sq->sq_callbpend;
7324 7416 sq->sq_callbpend = cbp;
7325 7417 return (cbp);
7326 7418 }
7327 7419
7328 7420 void
7329 7421 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7330 7422 {
7331 7423 callbparams_t **pp, *p;
7332 7424
7333 7425 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7334 7426
7335 7427 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7336 7428 if (p == cbp) {
7337 7429 *pp = p->cbp_next;
7338 7430 kmem_free(p, p->cbp_size);
7339 7431 return;
7340 7432 }
7341 7433 }
7342 7434 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7343 7435 "callbparams_free: not found\n"));
7344 7436 }
7345 7437
7346 7438 void
7347 7439 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7348 7440 {
7349 7441 callbparams_t **pp, *p;
7350 7442
7351 7443 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7352 7444
7353 7445 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7354 7446 if (p->cbp_id == id && p->cbp_flags == flag) {
7355 7447 *pp = p->cbp_next;
7356 7448 kmem_free(p, p->cbp_size);
7357 7449 return;
7358 7450 }
7359 7451 }
7360 7452 (void) (STRLOG(0, 0, 0, SL_CONSOLE,
7361 7453 "callbparams_free_id: not found\n"));
7362 7454 }
7363 7455
7364 7456 /*
7365 7457 * Callback wrapper function used by once-only callbacks that can be
7366 7458 * cancelled (qtimeout and qbufcall)
7367 7459 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7368 7460 * cancelled by the qun* functions.
7369 7461 */
7370 7462 void
7371 7463 qcallbwrapper(void *arg)
7372 7464 {
7373 7465 callbparams_t *cbp = arg;
7374 7466 syncq_t *sq;
7375 7467 uint16_t count = 0;
7376 7468 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7377 7469 uint16_t type;
7378 7470
7379 7471 sq = cbp->cbp_sq;
7380 7472 mutex_enter(SQLOCK(sq));
7381 7473 type = sq->sq_type;
7382 7474 if (!(type & SQ_CICB)) {
7383 7475 count = sq->sq_count;
7384 7476 SQ_PUTLOCKS_ENTER(sq);
7385 7477 SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7386 7478 SUM_SQ_PUTCOUNTS(sq, count);
7387 7479 sq->sq_needexcl++;
7388 7480 ASSERT(sq->sq_needexcl != 0); /* wraparound */
7389 7481 waitflags |= SQ_MESSAGES;
7390 7482 }
7391 7483 /* Can not handle exclusive entry at outer perimeter */
7392 7484 ASSERT(type & SQ_COCB);
7393 7485
7394 7486 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7395 7487 if ((sq->sq_callbflags & cbp->cbp_flags) &&
7396 7488 (sq->sq_cancelid == cbp->cbp_id)) {
7397 7489 /* timeout has been cancelled */
7398 7490 sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7399 7491 callbparams_free(sq, cbp);
7400 7492 if (!(type & SQ_CICB)) {
7401 7493 ASSERT(sq->sq_needexcl > 0);
7402 7494 sq->sq_needexcl--;
7403 7495 if (sq->sq_needexcl == 0) {
7404 7496 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7405 7497 }
7406 7498 SQ_PUTLOCKS_EXIT(sq);
7407 7499 }
7408 7500 mutex_exit(SQLOCK(sq));
7409 7501 return;
7410 7502 }
7411 7503 sq->sq_flags |= SQ_WANTWAKEUP;
7412 7504 if (!(type & SQ_CICB)) {
7413 7505 SQ_PUTLOCKS_EXIT(sq);
7414 7506 }
7415 7507 cv_wait(&sq->sq_wait, SQLOCK(sq));
7416 7508 if (!(type & SQ_CICB)) {
7417 7509 count = sq->sq_count;
7418 7510 SQ_PUTLOCKS_ENTER(sq);
7419 7511 SUM_SQ_PUTCOUNTS(sq, count);
7420 7512 }
7421 7513 }
7422 7514
7423 7515 sq->sq_count++;
7424 7516 ASSERT(sq->sq_count != 0); /* Wraparound */
7425 7517 if (!(type & SQ_CICB)) {
7426 7518 ASSERT(count == 0);
7427 7519 sq->sq_flags |= SQ_EXCL;
7428 7520 ASSERT(sq->sq_needexcl > 0);
7429 7521 sq->sq_needexcl--;
7430 7522 if (sq->sq_needexcl == 0) {
7431 7523 SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7432 7524 }
7433 7525 SQ_PUTLOCKS_EXIT(sq);
7434 7526 }
7435 7527
7436 7528 mutex_exit(SQLOCK(sq));
7437 7529
7438 7530 cbp->cbp_func(cbp->cbp_arg);
7439 7531
7440 7532 /*
7441 7533 * We drop the lock only for leavesq to re-acquire it.
7442 7534 * Possible optimization is inline of leavesq.
7443 7535 */
7444 7536 mutex_enter(SQLOCK(sq));
7445 7537 callbparams_free(sq, cbp);
7446 7538 mutex_exit(SQLOCK(sq));
7447 7539 leavesq(sq, SQ_CALLBACK);
7448 7540 }
7449 7541
7450 7542 /*
7451 7543 * No need to grab sq_putlocks here. See comment in strsubr.h that
7452 7544 * explains when sq_putlocks are used.
7453 7545 *
7454 7546 * sq_count (or one of the sq_putcounts) has already been
7455 7547 * decremented by the caller, and if SQ_QUEUED, we need to call
7456 7548 * drain_syncq (the global syncq drain).
7457 7549 * If putnext_tail is called with the SQ_EXCL bit set, we are in
7458 7550 * one of two states, non-CIPUT perimeter, and we need to clear
7459 7551 * it, or we went exclusive in the put procedure. In any case,
7460 7552 * we want to clear the bit now, and it is probably easier to do
7461 7553 * this at the beginning of this function (remember, we hold
7462 7554 * the SQLOCK). Lastly, if there are other messages queued
7463 7555 * on the syncq (and not for our destination), enable the syncq
7464 7556 * for background work.
7465 7557 */
7466 7558
7467 7559 /* ARGSUSED */
7468 7560 void
7469 7561 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7470 7562 {
7471 7563 uint16_t flags = sq->sq_flags;
7472 7564
7473 7565 ASSERT(MUTEX_HELD(SQLOCK(sq)));
7474 7566 ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7475 7567
7476 7568 /* Clear SQ_EXCL if set in passflags */
7477 7569 if (passflags & SQ_EXCL) {
7478 7570 flags &= ~SQ_EXCL;
7479 7571 }
7480 7572 if (flags & SQ_WANTWAKEUP) {
7481 7573 flags &= ~SQ_WANTWAKEUP;
7482 7574 cv_broadcast(&sq->sq_wait);
7483 7575 }
7484 7576 if (flags & SQ_WANTEXWAKEUP) {
7485 7577 flags &= ~SQ_WANTEXWAKEUP;
7486 7578 cv_broadcast(&sq->sq_exitwait);
7487 7579 }
7488 7580 sq->sq_flags = flags;
7489 7581
7490 7582 /*
7491 7583 * We have cleared SQ_EXCL if we were asked to, and started
7492 7584 * the wakeup process for waiters. If there are no writers
7493 7585 * then we need to drain the syncq if we were told to, or
7494 7586 * enable the background thread to do it.
7495 7587 */
7496 7588 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7497 7589 if ((passflags & SQ_QUEUED) ||
7498 7590 (sq->sq_svcflags & SQ_DISABLED)) {
7499 7591 /* drain_syncq will take care of events in the list */
7500 7592 drain_syncq(sq);
7501 7593 return;
7502 7594 } else if (flags & SQ_QUEUED) {
7503 7595 sqenable(sq);
7504 7596 }
7505 7597 }
7506 7598 /* Drop the SQLOCK on exit */
7507 7599 mutex_exit(SQLOCK(sq));
7508 7600 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7509 7601 "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7510 7602 }
7511 7603
7512 7604 void
7513 7605 set_qend(queue_t *q)
7514 7606 {
7515 7607 mutex_enter(QLOCK(q));
7516 7608 if (!O_SAMESTR(q))
7517 7609 q->q_flag |= QEND;
7518 7610 else
7519 7611 q->q_flag &= ~QEND;
7520 7612 mutex_exit(QLOCK(q));
7521 7613 q = _OTHERQ(q);
7522 7614 mutex_enter(QLOCK(q));
7523 7615 if (!O_SAMESTR(q))
7524 7616 q->q_flag |= QEND;
7525 7617 else
7526 7618 q->q_flag &= ~QEND;
7527 7619 mutex_exit(QLOCK(q));
7528 7620 }
7529 7621
7530 7622 /*
7531 7623 * Set QFULL in next service procedure queue (that cares) if not already
7532 7624 * set and if there are already more messages on the syncq than
7533 7625 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on
7534 7626 * any syncq.
7535 7627 *
7536 7628 * The fq here is the next queue with a service procedure. This is where
7537 7629 * we would fail canputnext, so this is where we need to set QFULL.
7538 7630 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7539 7631 *
7540 7632 * We already have QLOCK at this point. To avoid cross-locks with
7541 7633 * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7542 7634 * SQLOCK and sd_reflock, we need to drop respective locks first.
7543 7635 */
7544 7636 void
7545 7637 set_qfull(queue_t *q)
7546 7638 {
7547 7639 queue_t *fq = NULL;
7548 7640
7549 7641 ASSERT(MUTEX_HELD(QLOCK(q)));
7550 7642 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7551 7643 (q->q_syncqmsgs > sq_max_size)) {
7552 7644 if ((fq = q->q_nfsrv) == q) {
7553 7645 fq->q_flag |= QFULL;
7554 7646 } else {
7555 7647 mutex_exit(QLOCK(q));
7556 7648 mutex_enter(QLOCK(fq));
7557 7649 fq->q_flag |= QFULL;
7558 7650 mutex_exit(QLOCK(fq));
7559 7651 mutex_enter(QLOCK(q));
7560 7652 }
7561 7653 }
7562 7654 }
7563 7655
7564 7656 void
7565 7657 clr_qfull(queue_t *q)
7566 7658 {
7567 7659 queue_t *oq = q;
7568 7660
7569 7661 q = q->q_nfsrv;
7570 7662 /* Fast check if there is any work to do before getting the lock. */
7571 7663 if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7572 7664 return;
7573 7665 }
7574 7666
7575 7667 /*
7576 7668 * Do not reset QFULL (and backenable) if the q_count is the reason
7577 7669 * for QFULL being set.
7578 7670 */
7579 7671 mutex_enter(QLOCK(q));
7580 7672 /*
7581 7673 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7582 7674 * Hence clear the QFULL.
7583 7675 * If both q_count and q_mblkcnt are less than the hiwat mark,
7584 7676 * clear the QFULL.
7585 7677 */
7586 7678 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7587 7679 (q->q_mblkcnt < q->q_hiwat))) {
7588 7680 q->q_flag &= ~QFULL;
7589 7681 /*
7590 7682 * A little more confusing, how about this way:
7591 7683 * if someone wants to write,
7592 7684 * AND
7593 7685 * both counts are less than the lowat mark
7594 7686 * OR
7595 7687 * the lowat mark is zero
7596 7688 * THEN
7597 7689 * backenable
7598 7690 */
7599 7691 if ((q->q_flag & QWANTW) &&
7600 7692 (((q->q_count < q->q_lowat) &&
7601 7693 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7602 7694 q->q_flag &= ~QWANTW;
7603 7695 mutex_exit(QLOCK(q));
7604 7696 backenable(oq, 0);
7605 7697 } else
7606 7698 mutex_exit(QLOCK(q));
7607 7699 } else
7608 7700 mutex_exit(QLOCK(q));
7609 7701 }
7610 7702
7611 7703 /*
7612 7704 * Set the forward service procedure pointer.
7613 7705 *
7614 7706 * Called at insert-time to cache a queue's next forward service procedure in
7615 7707 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted
7616 7708 * has a service procedure then q_nfsrv points to itself. If the queue to be
7617 7709 * inserted does not have a service procedure, then q_nfsrv points to the next
7618 7710 * queue forward that has a service procedure. If the queue is at the logical
7619 7711 * end of the stream (driver for write side, stream head for the read side)
7620 7712 * and does not have a service procedure, then q_nfsrv also points to itself.
7621 7713 */
7622 7714 void
7623 7715 set_nfsrv_ptr(
7624 7716 queue_t *rnew, /* read queue pointer to new module */
7625 7717 queue_t *wnew, /* write queue pointer to new module */
7626 7718 queue_t *prev_rq, /* read queue pointer to the module above */
7627 7719 queue_t *prev_wq) /* write queue pointer to the module above */
7628 7720 {
7629 7721 queue_t *qp;
7630 7722
7631 7723 if (prev_wq->q_next == NULL) {
7632 7724 /*
7633 7725 * Insert the driver, initialize the driver and stream head.
7634 7726 * In this case, prev_rq/prev_wq should be the stream head.
7635 7727 * _I_INSERT does not allow inserting a driver. Make sure
7636 7728 * that it is not an insertion.
7637 7729 */
7638 7730 ASSERT(!(rnew->q_flag & _QINSERTING));
7639 7731 wnew->q_nfsrv = wnew;
7640 7732 if (rnew->q_qinfo->qi_srvp)
7641 7733 rnew->q_nfsrv = rnew;
7642 7734 else
7643 7735 rnew->q_nfsrv = prev_rq;
7644 7736 prev_rq->q_nfsrv = prev_rq;
7645 7737 prev_wq->q_nfsrv = prev_wq;
7646 7738 } else {
7647 7739 /*
7648 7740 * set up read side q_nfsrv pointer. This MUST be done
7649 7741 * before setting the write side, because the setting of
7650 7742 * the write side for a fifo may depend on it.
7651 7743 *
7652 7744 * Suppose we have a fifo that only has pipemod pushed.
7653 7745 * pipemod has no read or write service procedures, so
7654 7746 * nfsrv for both pipemod queues points to prev_rq (the
7655 7747 * stream read head). Now push bufmod (which has only a
7656 7748 * read service procedure). Doing the write side first,
7657 7749 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7658 7750 * is WRONG; the next queue forward from wnew with a
7659 7751 * service procedure will be rnew, not the stream read head.
7660 7752 * Since the downstream queue (which in the case of a fifo
7661 7753 * is the read queue rnew) can affect upstream queues, it
7662 7754 * needs to be done first. Setting up the read side first
7663 7755 * sets nfsrv for both pipemod queues to rnew and then
7664 7756 * when the write side is set up, wnew-q_nfsrv will also
7665 7757 * point to rnew.
7666 7758 */
7667 7759 if (rnew->q_qinfo->qi_srvp) {
7668 7760 /*
7669 7761 * use _OTHERQ() because, if this is a pipe, next
7670 7762 * module may have been pushed from other end and
7671 7763 * q_next could be a read queue.
7672 7764 */
7673 7765 qp = _OTHERQ(prev_wq->q_next);
7674 7766 while (qp && qp->q_nfsrv != qp) {
7675 7767 qp->q_nfsrv = rnew;
7676 7768 qp = backq(qp);
7677 7769 }
7678 7770 rnew->q_nfsrv = rnew;
7679 7771 } else
7680 7772 rnew->q_nfsrv = prev_rq->q_nfsrv;
7681 7773
7682 7774 /* set up write side q_nfsrv pointer */
7683 7775 if (wnew->q_qinfo->qi_srvp) {
7684 7776 wnew->q_nfsrv = wnew;
7685 7777
7686 7778 /*
7687 7779 * For insertion, need to update nfsrv of the modules
7688 7780 * above which do not have a service routine.
7689 7781 */
7690 7782 if (rnew->q_flag & _QINSERTING) {
7691 7783 for (qp = prev_wq;
7692 7784 qp != NULL && qp->q_nfsrv != qp;
7693 7785 qp = backq(qp)) {
7694 7786 qp->q_nfsrv = wnew->q_nfsrv;
7695 7787 }
7696 7788 }
7697 7789 } else {
7698 7790 if (prev_wq->q_next == prev_rq)
7699 7791 /*
7700 7792 * Since prev_wq/prev_rq are the middle of a
7701 7793 * fifo, wnew/rnew will also be the middle of
7702 7794 * a fifo and wnew's nfsrv is same as rnew's.
7703 7795 */
7704 7796 wnew->q_nfsrv = rnew->q_nfsrv;
7705 7797 else
7706 7798 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7707 7799 }
7708 7800 }
7709 7801 }
7710 7802
7711 7803 /*
7712 7804 * Reset the forward service procedure pointer; called at remove-time.
7713 7805 */
7714 7806 void
7715 7807 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7716 7808 {
7717 7809 queue_t *tmp_qp;
7718 7810
7719 7811 /* Reset the write side q_nfsrv pointer for _I_REMOVE */
7720 7812 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7721 7813 for (tmp_qp = backq(wqp);
7722 7814 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7723 7815 tmp_qp = backq(tmp_qp)) {
7724 7816 tmp_qp->q_nfsrv = wqp->q_nfsrv;
7725 7817 }
7726 7818 }
7727 7819
7728 7820 /* reset the read side q_nfsrv pointer */
7729 7821 if (rqp->q_qinfo->qi_srvp) {
7730 7822 if (wqp->q_next) { /* non-driver case */
7731 7823 tmp_qp = _OTHERQ(wqp->q_next);
7732 7824 while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7733 7825 /* Note that rqp->q_next cannot be NULL */
7734 7826 ASSERT(rqp->q_next != NULL);
7735 7827 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7736 7828 tmp_qp = backq(tmp_qp);
7737 7829 }
7738 7830 }
7739 7831 }
7740 7832 }
7741 7833
7742 7834 /*
7743 7835 * This routine should be called after all stream geometry changes to update
7744 7836 * the stream head cached struio() rd/wr queue pointers. Note must be called
7745 7837 * with the streamlock()ed.
7746 7838 *
7747 7839 * Note: only enables Synchronous STREAMS for a side of a Stream which has
7748 7840 * an explicit synchronous barrier module queue. That is, a queue that
7749 7841 * has specified a struio() type.
7750 7842 */
7751 7843 static void
7752 7844 strsetuio(stdata_t *stp)
7753 7845 {
7754 7846 queue_t *wrq;
7755 7847
7756 7848 if (stp->sd_flag & STPLEX) {
7757 7849 /*
7758 7850 * Not streamhead, but a mux, so no Synchronous STREAMS.
7759 7851 */
7760 7852 stp->sd_struiowrq = NULL;
7761 7853 stp->sd_struiordq = NULL;
7762 7854 return;
7763 7855 }
7764 7856 /*
7765 7857 * Scan the write queue(s) while synchronous
7766 7858 * until we find a qinfo uio type specified.
7767 7859 */
7768 7860 wrq = stp->sd_wrq->q_next;
7769 7861 while (wrq) {
7770 7862 if (wrq->q_struiot == STRUIOT_NONE) {
7771 7863 wrq = 0;
7772 7864 break;
7773 7865 }
7774 7866 if (wrq->q_struiot != STRUIOT_DONTCARE)
7775 7867 break;
7776 7868 if (! _SAMESTR(wrq)) {
7777 7869 wrq = 0;
7778 7870 break;
7779 7871 }
7780 7872 wrq = wrq->q_next;
7781 7873 }
7782 7874 stp->sd_struiowrq = wrq;
7783 7875 /*
7784 7876 * Scan the read queue(s) while synchronous
7785 7877 * until we find a qinfo uio type specified.
7786 7878 */
7787 7879 wrq = stp->sd_wrq->q_next;
7788 7880 while (wrq) {
7789 7881 if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7790 7882 wrq = 0;
7791 7883 break;
7792 7884 }
7793 7885 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7794 7886 break;
7795 7887 if (! _SAMESTR(wrq)) {
7796 7888 wrq = 0;
7797 7889 break;
7798 7890 }
7799 7891 wrq = wrq->q_next;
7800 7892 }
7801 7893 stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7802 7894 }
7803 7895
7804 7896 /*
7805 7897 * pass_wput, unblocks the passthru queues, so that
7806 7898 * messages can arrive at muxs lower read queue, before
7807 7899 * I_LINK/I_UNLINK is acked/nacked.
7808 7900 */
7809 7901 static void
7810 7902 pass_wput(queue_t *q, mblk_t *mp)
7811 7903 {
7812 7904 syncq_t *sq;
7813 7905
7814 7906 sq = _RD(q)->q_syncq;
7815 7907 if (sq->sq_flags & SQ_BLOCKED)
7816 7908 unblocksq(sq, SQ_BLOCKED, 0);
7817 7909 putnext(q, mp);
7818 7910 }
7819 7911
7820 7912 /*
7821 7913 * Set up queues for the link/unlink.
7822 7914 * Create a new queue and block it and then insert it
7823 7915 * below the stream head on the lower stream.
7824 7916 * This prevents any messages from arriving during the setq
7825 7917 * as well as while the mux is processing the LINK/I_UNLINK.
7826 7918 * The blocked passq is unblocked once the LINK/I_UNLINK has
7827 7919 * been acked or nacked or if a message is generated and sent
7828 7920 * down muxs write put procedure.
7829 7921 * See pass_wput().
7830 7922 *
7831 7923 * After the new queue is inserted, all messages coming from below are
7832 7924 * blocked. The call to strlock will ensure that all activity in the stream head
7833 7925 * read queue syncq is stopped (sq_count drops to zero).
7834 7926 */
7835 7927 static queue_t *
7836 7928 link_addpassthru(stdata_t *stpdown)
7837 7929 {
7838 7930 queue_t *passq;
7839 7931 sqlist_t sqlist;
7840 7932
7841 7933 passq = allocq();
7842 7934 STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7843 7935 /* setq might sleep in allocator - avoid holding locks. */
7844 7936 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7845 7937 SQ_CI|SQ_CO, B_FALSE);
7846 7938 claimq(passq);
7847 7939 blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7848 7940 insertq(STREAM(passq), passq);
7849 7941
7850 7942 /*
7851 7943 * Use strlock() to wait for the stream head sq_count to drop to zero
7852 7944 * since we are going to change q_ptr in the stream head. Note that
7853 7945 * insertq() doesn't wait for any syncq counts to drop to zero.
7854 7946 */
7855 7947 sqlist.sqlist_head = NULL;
7856 7948 sqlist.sqlist_index = 0;
7857 7949 sqlist.sqlist_size = sizeof (sqlist_t);
7858 7950 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7859 7951 strlock(stpdown, &sqlist);
7860 7952 strunlock(stpdown, &sqlist);
7861 7953
7862 7954 releaseq(passq);
7863 7955 return (passq);
7864 7956 }
7865 7957
7866 7958 /*
7867 7959 * Let messages flow up into the mux by removing
7868 7960 * the passq.
7869 7961 */
7870 7962 static void
7871 7963 link_rempassthru(queue_t *passq)
7872 7964 {
7873 7965 claimq(passq);
7874 7966 removeq(passq);
7875 7967 releaseq(passq);
7876 7968 freeq(passq);
7877 7969 }
7878 7970
7879 7971 /*
7880 7972 * Wait for the condition variable pointed to by `cvp' to be signaled,
7881 7973 * or for `tim' milliseconds to elapse, whichever comes first. If `tim'
7882 7974 * is negative, then there is no time limit. If `nosigs' is non-zero,
7883 7975 * then the wait will be non-interruptible.
7884 7976 *
7885 7977 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7886 7978 */
7887 7979 clock_t
7888 7980 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7889 7981 {
7890 7982 clock_t ret;
7891 7983
7892 7984 if (tim < 0) {
7893 7985 if (nosigs) {
7894 7986 cv_wait(cvp, mp);
7895 7987 ret = 1;
7896 7988 } else {
7897 7989 ret = cv_wait_sig(cvp, mp);
7898 7990 }
7899 7991 } else if (tim > 0) {
7900 7992 /*
7901 7993 * convert milliseconds to clock ticks
7902 7994 */
7903 7995 if (nosigs) {
7904 7996 ret = cv_reltimedwait(cvp, mp,
7905 7997 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7906 7998 } else {
7907 7999 ret = cv_reltimedwait_sig(cvp, mp,
7908 8000 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7909 8001 }
7910 8002 } else {
7911 8003 ret = -1;
7912 8004 }
7913 8005 return (ret);
7914 8006 }
7915 8007
7916 8008 /*
7917 8009 * Wait until the stream head can determine if it is at the mark but
7918 8010 * don't wait forever to prevent a race condition between the "mark" state
7919 8011 * in the stream head and any mark state in the caller/user of this routine.
7920 8012 *
7921 8013 * This is used by sockets and for a socket it would be incorrect
7922 8014 * to return a failure for SIOCATMARK when there is no data in the receive
7923 8015 * queue and the marked urgent data is traveling up the stream.
7924 8016 *
7925 8017 * This routine waits until the mark is known by waiting for one of these
7926 8018 * three events:
7927 8019 * The stream head read queue becoming non-empty (including an EOF).
7928 8020 * The STRATMARK flag being set (due to a MSGMARKNEXT message).
7929 8021 * The STRNOTATMARK flag being set (which indicates that the transport
7930 8022 * has sent a MSGNOTMARKNEXT message to indicate that it is not at
7931 8023 * the mark).
7932 8024 *
7933 8025 * The routine returns 1 if the stream is at the mark; 0 if it can
7934 8026 * be determined that the stream is not at the mark.
7935 8027 * If the wait times out and it can't determine
7936 8028 * whether or not the stream might be at the mark the routine will return -1.
7937 8029 *
7938 8030 * Note: This routine should only be used when a mark is pending i.e.,
7939 8031 * in the socket case the SIGURG has been posted.
7940 8032 * Note2: This can not wakeup just because synchronous streams indicate
7941 8033 * that data is available since it is not possible to use the synchronous
7942 8034 * streams interfaces to determine the b_flag value for the data queued below
7943 8035 * the stream head.
7944 8036 */
7945 8037 int
7946 8038 strwaitmark(vnode_t *vp)
7947 8039 {
7948 8040 struct stdata *stp = vp->v_stream;
7949 8041 queue_t *rq = _RD(stp->sd_wrq);
7950 8042 int mark;
7951 8043
7952 8044 mutex_enter(&stp->sd_lock);
7953 8045 while (rq->q_first == NULL &&
7954 8046 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7955 8047 stp->sd_flag |= RSLEEP;
7956 8048
7957 8049 /* Wait for 100 milliseconds for any state change. */
7958 8050 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7959 8051 mutex_exit(&stp->sd_lock);
7960 8052 return (-1);
7961 8053 }
7962 8054 }
7963 8055 if (stp->sd_flag & STRATMARK)
7964 8056 mark = 1;
7965 8057 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7966 8058 mark = 1;
7967 8059 else
7968 8060 mark = 0;
7969 8061
7970 8062 mutex_exit(&stp->sd_lock);
7971 8063 return (mark);
7972 8064 }
7973 8065
7974 8066 /*
7975 8067 * Set a read side error. If persist is set change the socket error
7976 8068 * to persistent. If errfunc is set install the function as the exported
7977 8069 * error handler.
7978 8070 */
7979 8071 void
7980 8072 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7981 8073 {
7982 8074 struct stdata *stp = vp->v_stream;
7983 8075
7984 8076 mutex_enter(&stp->sd_lock);
7985 8077 stp->sd_rerror = error;
7986 8078 if (error == 0 && errfunc == NULL)
7987 8079 stp->sd_flag &= ~STRDERR;
7988 8080 else
7989 8081 stp->sd_flag |= STRDERR;
7990 8082 if (persist) {
7991 8083 stp->sd_flag &= ~STRDERRNONPERSIST;
7992 8084 } else {
7993 8085 stp->sd_flag |= STRDERRNONPERSIST;
7994 8086 }
7995 8087 stp->sd_rderrfunc = errfunc;
7996 8088 if (error != 0 || errfunc != NULL) {
7997 8089 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
7998 8090 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
7999 8091 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8000 8092
8001 8093 mutex_exit(&stp->sd_lock);
8002 8094 pollwakeup(&stp->sd_pollist, POLLERR);
8003 8095 mutex_enter(&stp->sd_lock);
8004 8096
8005 8097 if (stp->sd_sigflags & S_ERROR)
8006 8098 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8007 8099 }
8008 8100 mutex_exit(&stp->sd_lock);
8009 8101 }
8010 8102
8011 8103 /*
8012 8104 * Set a write side error. If persist is set change the socket error
8013 8105 * to persistent.
8014 8106 */
8015 8107 void
8016 8108 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8017 8109 {
8018 8110 struct stdata *stp = vp->v_stream;
8019 8111
8020 8112 mutex_enter(&stp->sd_lock);
8021 8113 stp->sd_werror = error;
8022 8114 if (error == 0 && errfunc == NULL)
8023 8115 stp->sd_flag &= ~STWRERR;
8024 8116 else
8025 8117 stp->sd_flag |= STWRERR;
8026 8118 if (persist) {
8027 8119 stp->sd_flag &= ~STWRERRNONPERSIST;
8028 8120 } else {
8029 8121 stp->sd_flag |= STWRERRNONPERSIST;
8030 8122 }
8031 8123 stp->sd_wrerrfunc = errfunc;
8032 8124 if (error != 0 || errfunc != NULL) {
8033 8125 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */
8034 8126 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */
8035 8127 cv_broadcast(&stp->sd_monitor); /* ioctllers */
8036 8128
8037 8129 mutex_exit(&stp->sd_lock);
8038 8130 pollwakeup(&stp->sd_pollist, POLLERR);
8039 8131 mutex_enter(&stp->sd_lock);
8040 8132
8041 8133 if (stp->sd_sigflags & S_ERROR)
8042 8134 strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8043 8135 }
8044 8136 mutex_exit(&stp->sd_lock);
8045 8137 }
8046 8138
8047 8139 /*
8048 8140 * Make the stream return 0 (EOF) when all data has been read.
8049 8141 * No effect on write side.
8050 8142 */
8051 8143 void
8052 8144 strseteof(vnode_t *vp, int eof)
8053 8145 {
8054 8146 struct stdata *stp = vp->v_stream;
8055 8147
8056 8148 mutex_enter(&stp->sd_lock);
8057 8149 if (!eof) {
8058 8150 stp->sd_flag &= ~STREOF;
8059 8151 mutex_exit(&stp->sd_lock);
8060 8152 return;
8061 8153 }
8062 8154 stp->sd_flag |= STREOF;
8063 8155 if (stp->sd_flag & RSLEEP) {
8064 8156 stp->sd_flag &= ~RSLEEP;
8065 8157 cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8066 8158 }
8067 8159
8068 8160 mutex_exit(&stp->sd_lock);
8069 8161 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8070 8162 mutex_enter(&stp->sd_lock);
8071 8163
8072 8164 if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8073 8165 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8074 8166 mutex_exit(&stp->sd_lock);
8075 8167 }
8076 8168
8077 8169 void
↓ open down ↓ |
4723 lines elided |
↑ open up ↑ |
8078 8170 strflushrq(vnode_t *vp, int flag)
8079 8171 {
8080 8172 struct stdata *stp = vp->v_stream;
8081 8173
8082 8174 mutex_enter(&stp->sd_lock);
8083 8175 flushq(_RD(stp->sd_wrq), flag);
8084 8176 mutex_exit(&stp->sd_lock);
8085 8177 }
8086 8178
8087 8179 void
8088 -strsetrputhooks(vnode_t *vp, uint_t flags,
8089 - msgfunc_t protofunc, msgfunc_t miscfunc)
8180 +strsetrputhooks(vnode_t *vp, uint_t flags, msgfunc_t protofunc,
8181 + msgfunc_t miscfunc)
8090 8182 {
8091 8183 struct stdata *stp = vp->v_stream;
8092 8184
8093 8185 mutex_enter(&stp->sd_lock);
8094 8186
8095 8187 if (protofunc == NULL)
8096 8188 stp->sd_rprotofunc = strrput_proto;
8097 8189 else
8098 8190 stp->sd_rprotofunc = protofunc;
8099 8191
8100 8192 if (miscfunc == NULL)
8101 8193 stp->sd_rmiscfunc = strrput_misc;
8102 8194 else
8103 8195 stp->sd_rmiscfunc = miscfunc;
8104 8196
8105 8197 if (flags & SH_CONSOL_DATA)
8106 8198 stp->sd_rput_opt |= SR_CONSOL_DATA;
8107 8199 else
8108 8200 stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8109 8201
8110 8202 if (flags & SH_SIGALLDATA)
8111 8203 stp->sd_rput_opt |= SR_SIGALLDATA;
8112 8204 else
8113 8205 stp->sd_rput_opt &= ~SR_SIGALLDATA;
8114 8206
8115 8207 if (flags & SH_IGN_ZEROLEN)
8116 8208 stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8117 8209 else
8118 8210 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8119 8211
8120 8212 mutex_exit(&stp->sd_lock);
8121 8213 }
8122 8214
8123 8215 void
8124 8216 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8125 8217 {
8126 8218 struct stdata *stp = vp->v_stream;
8127 8219
8128 8220 mutex_enter(&stp->sd_lock);
8129 8221 stp->sd_closetime = closetime;
8130 8222
8131 8223 if (flags & SH_SIGPIPE)
8132 8224 stp->sd_wput_opt |= SW_SIGPIPE;
8133 8225 else
8134 8226 stp->sd_wput_opt &= ~SW_SIGPIPE;
8135 8227 if (flags & SH_RECHECK_ERR)
8136 8228 stp->sd_wput_opt |= SW_RECHECK_ERR;
8137 8229 else
8138 8230 stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8139 8231
8140 8232 mutex_exit(&stp->sd_lock);
8141 8233 }
8142 8234
8143 8235 void
8144 8236 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8145 8237 {
8146 8238 struct stdata *stp = vp->v_stream;
8147 8239
8148 8240 mutex_enter(&stp->sd_lock);
8149 8241
8150 8242 stp->sd_rputdatafunc = rdatafunc;
8151 8243 stp->sd_wputdatafunc = wdatafunc;
8152 8244
8153 8245 mutex_exit(&stp->sd_lock);
8154 8246 }
8155 8247
8156 8248 /* Used within framework when the queue is already locked */
8157 8249 void
8158 8250 qenable_locked(queue_t *q)
8159 8251 {
8160 8252 stdata_t *stp = STREAM(q);
8161 8253
8162 8254 ASSERT(MUTEX_HELD(QLOCK(q)));
8163 8255
8164 8256 if (!q->q_qinfo->qi_srvp)
8165 8257 return;
8166 8258
8167 8259 /*
8168 8260 * Do not place on run queue if already enabled or closing.
8169 8261 */
8170 8262 if (q->q_flag & (QWCLOSE|QENAB))
8171 8263 return;
8172 8264
8173 8265 /*
8174 8266 * mark queue enabled and place on run list if it is not already being
8175 8267 * serviced. If it is serviced, the runservice() function will detect
8176 8268 * that QENAB is set and call service procedure before clearing
8177 8269 * QINSERVICE flag.
8178 8270 */
8179 8271 q->q_flag |= QENAB;
8180 8272 if (q->q_flag & QINSERVICE)
8181 8273 return;
8182 8274
8183 8275 /* Record the time of qenable */
8184 8276 q->q_qtstamp = ddi_get_lbolt();
8185 8277
8186 8278 /*
8187 8279 * Put the queue in the stp list and schedule it for background
8188 8280 * processing if it is not already scheduled or if stream head does not
8189 8281 * intent to process it in the foreground later by setting
8190 8282 * STRS_WILLSERVICE flag.
8191 8283 */
8192 8284 mutex_enter(&stp->sd_qlock);
8193 8285 /*
8194 8286 * If there are already something on the list, stp flags should show
8195 8287 * intention to drain it.
8196 8288 */
8197 8289 IMPLY(STREAM_NEEDSERVICE(stp),
8198 8290 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8199 8291
8200 8292 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8201 8293 stp->sd_nqueues++;
8202 8294
8203 8295 /*
8204 8296 * If no one will drain this stream we are the first producer and
8205 8297 * need to schedule it for background thread.
8206 8298 */
8207 8299 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8208 8300 /*
8209 8301 * No one will service this stream later, so we have to
8210 8302 * schedule it now.
8211 8303 */
8212 8304 STRSTAT(stenables);
8213 8305 stp->sd_svcflags |= STRS_SCHEDULED;
8214 8306 stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8215 8307 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8216 8308
8217 8309 if (stp->sd_servid == NULL) {
8218 8310 /*
8219 8311 * Task queue failed so fail over to the backup
8220 8312 * servicing thread.
8221 8313 */
8222 8314 STRSTAT(taskqfails);
8223 8315 /*
8224 8316 * It is safe to clear STRS_SCHEDULED flag because it
8225 8317 * was set by this thread above.
8226 8318 */
8227 8319 stp->sd_svcflags &= ~STRS_SCHEDULED;
8228 8320
8229 8321 /*
8230 8322 * Failover scheduling is protected by service_queue
8231 8323 * lock.
8232 8324 */
8233 8325 mutex_enter(&service_queue);
8234 8326 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8235 8327 ASSERT(q->q_link == NULL);
8236 8328 /*
8237 8329 * Append the queue to qhead/qtail list.
8238 8330 */
8239 8331 if (qhead == NULL)
8240 8332 qhead = q;
8241 8333 else
8242 8334 qtail->q_link = q;
8243 8335 qtail = q;
8244 8336 /*
8245 8337 * Clear stp queue list.
8246 8338 */
8247 8339 stp->sd_qhead = stp->sd_qtail = NULL;
8248 8340 stp->sd_nqueues = 0;
8249 8341 /*
8250 8342 * Wakeup background queue processing thread.
8251 8343 */
8252 8344 cv_signal(&services_to_run);
8253 8345 mutex_exit(&service_queue);
8254 8346 }
8255 8347 }
8256 8348 mutex_exit(&stp->sd_qlock);
8257 8349 }
8258 8350
8259 8351 static void
8260 8352 queue_service(queue_t *q)
8261 8353 {
8262 8354 /*
8263 8355 * The queue in the list should have
8264 8356 * QENAB flag set and should not have
8265 8357 * QINSERVICE flag set. QINSERVICE is
8266 8358 * set when the queue is dequeued and
8267 8359 * qenable_locked doesn't enqueue a
8268 8360 * queue with QINSERVICE set.
8269 8361 */
8270 8362
8271 8363 ASSERT(!(q->q_flag & QINSERVICE));
8272 8364 ASSERT((q->q_flag & QENAB));
8273 8365 mutex_enter(QLOCK(q));
8274 8366 q->q_flag &= ~QENAB;
8275 8367 q->q_flag |= QINSERVICE;
8276 8368 mutex_exit(QLOCK(q));
8277 8369 runservice(q);
8278 8370 }
8279 8371
8280 8372 static void
8281 8373 syncq_service(syncq_t *sq)
8282 8374 {
8283 8375 STRSTAT(syncqservice);
8284 8376 mutex_enter(SQLOCK(sq));
8285 8377 ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8286 8378 ASSERT(sq->sq_servcount != 0);
8287 8379 ASSERT(sq->sq_next == NULL);
8288 8380
8289 8381 /* if we came here from the background thread, clear the flag */
8290 8382 if (sq->sq_svcflags & SQ_BGTHREAD)
8291 8383 sq->sq_svcflags &= ~SQ_BGTHREAD;
8292 8384
8293 8385 /* let drain_syncq know that it's being called in the background */
8294 8386 sq->sq_svcflags |= SQ_SERVICE;
8295 8387 drain_syncq(sq);
8296 8388 }
8297 8389
8298 8390 static void
8299 8391 qwriter_outer_service(syncq_t *outer)
8300 8392 {
8301 8393 /*
8302 8394 * Note that SQ_WRITER is used on the outer perimeter
8303 8395 * to signal that a qwriter(OUTER) is either investigating
8304 8396 * running or that it is actually running a function.
8305 8397 */
8306 8398 outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8307 8399
8308 8400 /*
8309 8401 * All inner syncq are empty and have SQ_WRITER set
8310 8402 * to block entering the outer perimeter.
8311 8403 *
8312 8404 * We do not need to explicitly call write_now since
8313 8405 * outer_exit does it for us.
8314 8406 */
8315 8407 outer_exit(outer);
8316 8408 }
8317 8409
8318 8410 static void
8319 8411 mblk_free(mblk_t *mp)
8320 8412 {
8321 8413 dblk_t *dbp = mp->b_datap;
8322 8414 frtn_t *frp = dbp->db_frtnp;
8323 8415
8324 8416 mp->b_next = NULL;
8325 8417 if (dbp->db_fthdr != NULL)
8326 8418 str_ftfree(dbp);
8327 8419
8328 8420 ASSERT(dbp->db_fthdr == NULL);
8329 8421 frp->free_func(frp->free_arg);
8330 8422 ASSERT(dbp->db_mblk == mp);
8331 8423
8332 8424 if (dbp->db_credp != NULL) {
8333 8425 crfree(dbp->db_credp);
8334 8426 dbp->db_credp = NULL;
8335 8427 }
8336 8428 dbp->db_cpid = -1;
8337 8429 dbp->db_struioflag = 0;
8338 8430 dbp->db_struioun.cksum.flags = 0;
8339 8431
8340 8432 kmem_cache_free(dbp->db_cache, dbp);
8341 8433 }
8342 8434
8343 8435 /*
8344 8436 * Background processing of the stream queue list.
8345 8437 */
8346 8438 static void
8347 8439 stream_service(stdata_t *stp)
8348 8440 {
8349 8441 queue_t *q;
8350 8442
8351 8443 mutex_enter(&stp->sd_qlock);
8352 8444
8353 8445 STR_SERVICE(stp, q);
8354 8446
8355 8447 stp->sd_svcflags &= ~STRS_SCHEDULED;
8356 8448 stp->sd_servid = NULL;
8357 8449 cv_signal(&stp->sd_qcv);
8358 8450 mutex_exit(&stp->sd_qlock);
8359 8451 }
8360 8452
8361 8453 /*
8362 8454 * Foreground processing of the stream queue list.
8363 8455 */
8364 8456 void
8365 8457 stream_runservice(stdata_t *stp)
8366 8458 {
8367 8459 queue_t *q;
8368 8460
8369 8461 mutex_enter(&stp->sd_qlock);
8370 8462 STRSTAT(rservice);
8371 8463 /*
8372 8464 * We are going to drain this stream queue list, so qenable_locked will
8373 8465 * not schedule it until we finish.
8374 8466 */
8375 8467 stp->sd_svcflags |= STRS_WILLSERVICE;
8376 8468
8377 8469 STR_SERVICE(stp, q);
8378 8470
8379 8471 stp->sd_svcflags &= ~STRS_WILLSERVICE;
8380 8472 mutex_exit(&stp->sd_qlock);
8381 8473 /*
8382 8474 * Help backup background thread to drain the qhead/qtail list.
8383 8475 */
8384 8476 while (qhead != NULL) {
8385 8477 STRSTAT(qhelps);
8386 8478 mutex_enter(&service_queue);
8387 8479 DQ(q, qhead, qtail, q_link);
8388 8480 mutex_exit(&service_queue);
8389 8481 if (q != NULL)
8390 8482 queue_service(q);
8391 8483 }
8392 8484 }
8393 8485
8394 8486 void
8395 8487 stream_willservice(stdata_t *stp)
8396 8488 {
8397 8489 mutex_enter(&stp->sd_qlock);
8398 8490 stp->sd_svcflags |= STRS_WILLSERVICE;
8399 8491 mutex_exit(&stp->sd_qlock);
8400 8492 }
8401 8493
8402 8494 /*
8403 8495 * Replace the cred currently in the mblk with a different one.
8404 8496 * Also update db_cpid.
8405 8497 */
8406 8498 void
8407 8499 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8408 8500 {
8409 8501 dblk_t *dbp = mp->b_datap;
8410 8502 cred_t *ocr = dbp->db_credp;
8411 8503
8412 8504 ASSERT(cr != NULL);
8413 8505
8414 8506 if (cr != ocr) {
8415 8507 crhold(dbp->db_credp = cr);
8416 8508 if (ocr != NULL)
8417 8509 crfree(ocr);
8418 8510 }
8419 8511 /* Don't overwrite with NOPID */
8420 8512 if (cpid != NOPID)
8421 8513 dbp->db_cpid = cpid;
8422 8514 }
8423 8515
8424 8516 /*
8425 8517 * If the src message has a cred, then replace the cred currently in the mblk
8426 8518 * with it.
8427 8519 * Also update db_cpid.
8428 8520 */
8429 8521 void
8430 8522 mblk_copycred(mblk_t *mp, const mblk_t *src)
8431 8523 {
8432 8524 dblk_t *dbp = mp->b_datap;
8433 8525 cred_t *cr, *ocr;
8434 8526 pid_t cpid;
8435 8527
8436 8528 cr = msg_getcred(src, &cpid);
8437 8529 if (cr == NULL)
8438 8530 return;
8439 8531
8440 8532 ocr = dbp->db_credp;
8441 8533 if (cr != ocr) {
8442 8534 crhold(dbp->db_credp = cr);
8443 8535 if (ocr != NULL)
8444 8536 crfree(ocr);
8445 8537 }
8446 8538 /* Don't overwrite with NOPID */
8447 8539 if (cpid != NOPID)
8448 8540 dbp->db_cpid = cpid;
8449 8541 }
8450 8542
8451 8543 int
8452 8544 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8453 8545 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8454 8546 uint32_t flags, int km_flags)
8455 8547 {
8456 8548 int rc = 0;
8457 8549
8458 8550 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8459 8551 if (mp->b_datap->db_type == M_DATA) {
8460 8552 /* Associate values for M_DATA type */
8461 8553 DB_CKSUMSTART(mp) = (intptr_t)start;
8462 8554 DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8463 8555 DB_CKSUMEND(mp) = (intptr_t)end;
8464 8556 DB_CKSUMFLAGS(mp) = flags;
8465 8557 DB_CKSUM16(mp) = (uint16_t)value;
8466 8558
8467 8559 } else {
8468 8560 pattrinfo_t pa_info;
8469 8561
8470 8562 ASSERT(mmd != NULL);
8471 8563
8472 8564 pa_info.type = PATTR_HCKSUM;
8473 8565 pa_info.len = sizeof (pattr_hcksum_t);
8474 8566
8475 8567 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8476 8568 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8477 8569
8478 8570 hck->hcksum_start_offset = start;
8479 8571 hck->hcksum_stuff_offset = stuff;
8480 8572 hck->hcksum_end_offset = end;
8481 8573 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8482 8574 hck->hcksum_flags = flags;
8483 8575 } else {
8484 8576 rc = -1;
8485 8577 }
8486 8578 }
8487 8579 return (rc);
8488 8580 }
8489 8581
8490 8582 void
8491 8583 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8492 8584 uint32_t *start, uint32_t *stuff, uint32_t *end,
8493 8585 uint32_t *value, uint32_t *flags)
8494 8586 {
8495 8587 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8496 8588 if (mp->b_datap->db_type == M_DATA) {
8497 8589 if (flags != NULL) {
8498 8590 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8499 8591 if ((*flags & (HCK_PARTIALCKSUM |
8500 8592 HCK_FULLCKSUM)) != 0) {
8501 8593 if (value != NULL)
8502 8594 *value = (uint32_t)DB_CKSUM16(mp);
8503 8595 if ((*flags & HCK_PARTIALCKSUM) != 0) {
8504 8596 if (start != NULL)
8505 8597 *start =
8506 8598 (uint32_t)DB_CKSUMSTART(mp);
8507 8599 if (stuff != NULL)
8508 8600 *stuff =
8509 8601 (uint32_t)DB_CKSUMSTUFF(mp);
8510 8602 if (end != NULL)
8511 8603 *end =
8512 8604 (uint32_t)DB_CKSUMEND(mp);
8513 8605 }
8514 8606 }
8515 8607 }
8516 8608 } else {
8517 8609 pattrinfo_t hck_attr = {PATTR_HCKSUM};
8518 8610
8519 8611 ASSERT(mmd != NULL);
8520 8612
8521 8613 /* get hardware checksum attribute */
8522 8614 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8523 8615 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8524 8616
8525 8617 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8526 8618 if (flags != NULL)
8527 8619 *flags = hck->hcksum_flags;
8528 8620 if (start != NULL)
8529 8621 *start = hck->hcksum_start_offset;
8530 8622 if (stuff != NULL)
8531 8623 *stuff = hck->hcksum_stuff_offset;
8532 8624 if (end != NULL)
8533 8625 *end = hck->hcksum_end_offset;
8534 8626 if (value != NULL)
8535 8627 *value = (uint32_t)
8536 8628 hck->hcksum_cksum_val.inet_cksum;
8537 8629 }
8538 8630 }
8539 8631 }
8540 8632
8541 8633 void
8542 8634 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8543 8635 {
8544 8636 ASSERT(DB_TYPE(mp) == M_DATA);
8545 8637 ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8546 8638
8547 8639 /* Set the flags */
8548 8640 DB_LSOFLAGS(mp) |= flags;
8549 8641 DB_LSOMSS(mp) = mss;
8550 8642 }
8551 8643
8552 8644 void
8553 8645 lso_info_cleanup(mblk_t *mp)
8554 8646 {
8555 8647 ASSERT(DB_TYPE(mp) == M_DATA);
8556 8648
8557 8649 /* Clear the flags */
8558 8650 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8559 8651 DB_LSOMSS(mp) = 0;
8560 8652 }
8561 8653
8562 8654 /*
8563 8655 * Checksum buffer *bp for len bytes with psum partial checksum,
8564 8656 * or 0 if none, and return the 16 bit partial checksum.
8565 8657 */
8566 8658 unsigned
8567 8659 bcksum(uchar_t *bp, int len, unsigned int psum)
8568 8660 {
8569 8661 int odd = len & 1;
8570 8662 extern unsigned int ip_ocsum();
8571 8663
8572 8664 if (((intptr_t)bp & 1) == 0 && !odd) {
8573 8665 /*
8574 8666 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8575 8667 */
8576 8668 return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8577 8669 }
8578 8670 if (((intptr_t)bp & 1) != 0) {
8579 8671 /*
8580 8672 * Bp isn't 16 bit aligned.
8581 8673 */
8582 8674 unsigned int tsum;
8583 8675
8584 8676 #ifdef _LITTLE_ENDIAN
8585 8677 psum += *bp;
8586 8678 #else
8587 8679 psum += *bp << 8;
8588 8680 #endif
8589 8681 len--;
8590 8682 bp++;
8591 8683 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8592 8684 psum += (tsum << 8) & 0xffff | (tsum >> 8);
8593 8685 if (len & 1) {
8594 8686 bp += len - 1;
8595 8687 #ifdef _LITTLE_ENDIAN
8596 8688 psum += *bp << 8;
8597 8689 #else
8598 8690 psum += *bp;
8599 8691 #endif
8600 8692 }
8601 8693 } else {
8602 8694 /*
8603 8695 * Bp is 16 bit aligned.
8604 8696 */
8605 8697 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8606 8698 if (odd) {
8607 8699 bp += len - 1;
8608 8700 #ifdef _LITTLE_ENDIAN
8609 8701 psum += *bp;
8610 8702 #else
8611 8703 psum += *bp << 8;
8612 8704 #endif
8613 8705 }
8614 8706 }
8615 8707 /*
8616 8708 * Normalize psum to 16 bits before returning the new partial
8617 8709 * checksum. The max psum value before normalization is 0x3FDFE.
8618 8710 */
8619 8711 return ((psum >> 16) + (psum & 0xFFFF));
8620 8712 }
8621 8713
8622 8714 boolean_t
8623 8715 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8624 8716 {
8625 8717 boolean_t rc;
8626 8718
8627 8719 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8628 8720 if (DB_TYPE(mp) == M_DATA) {
8629 8721 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8630 8722 } else {
8631 8723 pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8632 8724
8633 8725 ASSERT(mmd != NULL);
8634 8726 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8635 8727 }
8636 8728 return (rc);
8637 8729 }
8638 8730
8639 8731 void
8640 8732 freemsgchain(mblk_t *mp)
8641 8733 {
8642 8734 mblk_t *next;
8643 8735
8644 8736 while (mp != NULL) {
8645 8737 next = mp->b_next;
8646 8738 mp->b_next = NULL;
8647 8739
8648 8740 freemsg(mp);
8649 8741 mp = next;
8650 8742 }
8651 8743 }
8652 8744
8653 8745 mblk_t *
8654 8746 copymsgchain(mblk_t *mp)
8655 8747 {
8656 8748 mblk_t *nmp = NULL;
8657 8749 mblk_t **nmpp = &nmp;
8658 8750
8659 8751 for (; mp != NULL; mp = mp->b_next) {
8660 8752 if ((*nmpp = copymsg(mp)) == NULL) {
8661 8753 freemsgchain(nmp);
8662 8754 return (NULL);
8663 8755 }
8664 8756
8665 8757 nmpp = &((*nmpp)->b_next);
8666 8758 }
8667 8759
8668 8760 return (nmp);
8669 8761 }
8670 8762
8671 8763 /* NOTE: Do not add code after this point. */
8672 8764 #undef QLOCK
8673 8765
8674 8766 /*
8675 8767 * Replacement for QLOCK macro for those that can't use it.
8676 8768 */
8677 8769 kmutex_t *
8678 8770 QLOCK(queue_t *q)
8679 8771 {
8680 8772 return (&(q)->q_lock);
8681 8773 }
8682 8774
8683 8775 /*
8684 8776 * Dummy runqueues/queuerun functions functions for backwards compatibility.
8685 8777 */
8686 8778 #undef runqueues
8687 8779 void
8688 8780 runqueues(void)
8689 8781 {
8690 8782 }
8691 8783
8692 8784 #undef queuerun
8693 8785 void
8694 8786 queuerun(void)
8695 8787 {
8696 8788 }
8697 8789
8698 8790 /*
8699 8791 * Initialize the STR stack instance, which tracks autopush and persistent
8700 8792 * links.
8701 8793 */
8702 8794 /* ARGSUSED */
8703 8795 static void *
8704 8796 str_stack_init(netstackid_t stackid, netstack_t *ns)
8705 8797 {
8706 8798 str_stack_t *ss;
8707 8799 int i;
8708 8800
8709 8801 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8710 8802 ss->ss_netstack = ns;
8711 8803
8712 8804 /*
8713 8805 * set up autopush
8714 8806 */
8715 8807 sad_initspace(ss);
8716 8808
8717 8809 /*
8718 8810 * set up mux_node structures.
8719 8811 */
8720 8812 ss->ss_devcnt = devcnt; /* In case it should change before free */
8721 8813 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8722 8814 ss->ss_devcnt), KM_SLEEP);
8723 8815 for (i = 0; i < ss->ss_devcnt; i++)
8724 8816 ss->ss_mux_nodes[i].mn_imaj = i;
8725 8817 return (ss);
8726 8818 }
8727 8819
8728 8820 /*
8729 8821 * Note: run at zone shutdown and not destroy so that the PLINKs are
8730 8822 * gone by the time other cleanup happens from the destroy callbacks.
8731 8823 */
8732 8824 static void
8733 8825 str_stack_shutdown(netstackid_t stackid, void *arg)
8734 8826 {
8735 8827 str_stack_t *ss = (str_stack_t *)arg;
8736 8828 int i;
8737 8829 cred_t *cr;
8738 8830
8739 8831 cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8740 8832 ASSERT(cr != NULL);
8741 8833
8742 8834 /* Undo all the I_PLINKs for this zone */
8743 8835 for (i = 0; i < ss->ss_devcnt; i++) {
8744 8836 struct mux_edge *ep;
8745 8837 ldi_handle_t lh;
8746 8838 ldi_ident_t li;
8747 8839 int ret;
8748 8840 int rval;
8749 8841 dev_t rdev;
8750 8842
8751 8843 ep = ss->ss_mux_nodes[i].mn_outp;
8752 8844 if (ep == NULL)
8753 8845 continue;
8754 8846 ret = ldi_ident_from_major((major_t)i, &li);
8755 8847 if (ret != 0) {
8756 8848 continue;
8757 8849 }
8758 8850 rdev = ep->me_dev;
8759 8851 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8760 8852 cr, &lh, li);
8761 8853 if (ret != 0) {
8762 8854 ldi_ident_release(li);
8763 8855 continue;
8764 8856 }
8765 8857
8766 8858 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8767 8859 cr, &rval);
8768 8860 if (ret) {
8769 8861 (void) ldi_close(lh, FREAD|FWRITE, cr);
8770 8862 ldi_ident_release(li);
8771 8863 continue;
8772 8864 }
8773 8865 (void) ldi_close(lh, FREAD|FWRITE, cr);
8774 8866
8775 8867 /* Close layered handles */
8776 8868 ldi_ident_release(li);
8777 8869 }
8778 8870 crfree(cr);
8779 8871
8780 8872 sad_freespace(ss);
8781 8873
8782 8874 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8783 8875 ss->ss_mux_nodes = NULL;
8784 8876 }
8785 8877
8786 8878 /*
8787 8879 * Free the structure; str_stack_shutdown did the other cleanup work.
8788 8880 */
8789 8881 /* ARGSUSED */
8790 8882 static void
8791 8883 str_stack_fini(netstackid_t stackid, void *arg)
8792 8884 {
8793 8885 str_stack_t *ss = (str_stack_t *)arg;
8794 8886
8795 8887 kmem_free(ss, sizeof (*ss));
8796 8888 }
↓ open down ↓ |
697 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX