Print this page
fixup .text where possible
7127 remove -Wno-missing-braces from Makefile.uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/devpoll.c
+++ new/usr/src/uts/common/io/devpoll.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 27 * Copyright (c) 2012 by Delphix. All rights reserved.
28 28 * Copyright 2016 Joyent, Inc.
29 29 */
30 30
31 31 #include <sys/types.h>
32 32 #include <sys/devops.h>
33 33 #include <sys/conf.h>
34 34 #include <sys/modctl.h>
35 35 #include <sys/sunddi.h>
36 36 #include <sys/stat.h>
37 37 #include <sys/poll_impl.h>
38 38 #include <sys/errno.h>
39 39 #include <sys/kmem.h>
40 40 #include <sys/mkdev.h>
41 41 #include <sys/debug.h>
42 42 #include <sys/file.h>
43 43 #include <sys/sysmacros.h>
44 44 #include <sys/systm.h>
45 45 #include <sys/bitmap.h>
46 46 #include <sys/devpoll.h>
47 47 #include <sys/rctl.h>
48 48 #include <sys/resource.h>
49 49 #include <sys/schedctl.h>
50 50 #include <sys/epoll.h>
51 51
52 52 #define RESERVED 1
53 53
54 54 /* local data struct */
55 55 static dp_entry_t **devpolltbl; /* dev poll entries */
56 56 static size_t dptblsize;
57 57
58 58 static kmutex_t devpoll_lock; /* lock protecting dev tbl */
59 59 int devpoll_init; /* is /dev/poll initialized already */
60 60
61 61 /* device local functions */
62 62
63 63 static int dpopen(dev_t *devp, int flag, int otyp, cred_t *credp);
64 64 static int dpwrite(dev_t dev, struct uio *uiop, cred_t *credp);
65 65 static int dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
66 66 int *rvalp);
67 67 static int dppoll(dev_t dev, short events, int anyyet, short *reventsp,
68 68 struct pollhead **phpp);
69 69 static int dpclose(dev_t dev, int flag, int otyp, cred_t *credp);
70 70 static dev_info_t *dpdevi;
71 71
72 72
73 73 static struct cb_ops dp_cb_ops = {
74 74 dpopen, /* open */
75 75 dpclose, /* close */
76 76 nodev, /* strategy */
77 77 nodev, /* print */
78 78 nodev, /* dump */
79 79 nodev, /* read */
80 80 dpwrite, /* write */
81 81 dpioctl, /* ioctl */
82 82 nodev, /* devmap */
83 83 nodev, /* mmap */
84 84 nodev, /* segmap */
85 85 dppoll, /* poll */
86 86 ddi_prop_op, /* prop_op */
87 87 (struct streamtab *)0, /* streamtab */
88 88 D_MP, /* flags */
89 89 CB_REV, /* cb_ops revision */
90 90 nodev, /* aread */
91 91 nodev /* awrite */
92 92 };
93 93
94 94 static int dpattach(dev_info_t *, ddi_attach_cmd_t);
95 95 static int dpdetach(dev_info_t *, ddi_detach_cmd_t);
96 96 static int dpinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
97 97
98 98 static struct dev_ops dp_ops = {
99 99 DEVO_REV, /* devo_rev */
100 100 0, /* refcnt */
101 101 dpinfo, /* info */
102 102 nulldev, /* identify */
103 103 nulldev, /* probe */
104 104 dpattach, /* attach */
105 105 dpdetach, /* detach */
106 106 nodev, /* reset */
107 107 &dp_cb_ops, /* driver operations */
108 108 (struct bus_ops *)NULL, /* bus operations */
109 109 nulldev, /* power */
110 110 ddi_quiesce_not_needed, /* quiesce */
111 111 };
|
↓ open down ↓ |
111 lines elided |
↑ open up ↑ |
112 112
113 113
114 114 static struct modldrv modldrv = {
115 115 &mod_driverops, /* type of module - a driver */
116 116 "/dev/poll driver",
117 117 &dp_ops,
118 118 };
119 119
120 120 static struct modlinkage modlinkage = {
121 121 MODREV_1,
122 - (void *)&modldrv,
123 - NULL
122 + { (void *)&modldrv,
123 + NULL }
124 124 };
125 125
126 126 static void pcachelink_assoc(pollcache_t *, pollcache_t *);
127 127 static void pcachelink_mark_stale(pollcache_t *);
128 128 static void pcachelink_purge_stale(pollcache_t *);
129 129 static void pcachelink_purge_all(pollcache_t *);
130 130
131 131
132 132 /*
133 133 * Locking Design
134 134 *
135 135 * The /dev/poll driver shares most of its code with poll sys call whose
136 136 * code is in common/syscall/poll.c. In poll(2) design, the pollcache
137 137 * structure is per lwp. An implicit assumption is made there that some
138 138 * portion of pollcache will never be touched by other lwps. E.g., in
139 139 * poll(2) design, no lwp will ever need to grow bitmap of other lwp.
140 140 * This assumption is not true for /dev/poll; hence the need for extra
141 141 * locking.
142 142 *
143 143 * To allow more parallelism, each /dev/poll file descriptor (indexed by
144 144 * minor number) has its own lock. Since read (dpioctl) is a much more
145 145 * frequent operation than write, we want to allow multiple reads on same
146 146 * /dev/poll fd. However, we prevent writes from being starved by giving
147 147 * priority to write operation. Theoretically writes can starve reads as
148 148 * well. But in practical sense this is not important because (1) writes
149 149 * happens less often than reads, and (2) write operation defines the
150 150 * content of poll fd a cache set. If writes happens so often that they
151 151 * can starve reads, that means the cached set is very unstable. It may
152 152 * not make sense to read an unstable cache set anyway. Therefore, the
153 153 * writers starving readers case is not handled in this design.
154 154 */
155 155
156 156 int
157 157 _init()
158 158 {
159 159 int error;
160 160
161 161 dptblsize = DEVPOLLSIZE;
162 162 devpolltbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
163 163 mutex_init(&devpoll_lock, NULL, MUTEX_DEFAULT, NULL);
164 164 devpoll_init = 1;
165 165 if ((error = mod_install(&modlinkage)) != 0) {
166 166 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
167 167 devpoll_init = 0;
168 168 }
169 169 return (error);
170 170 }
171 171
172 172 int
173 173 _fini()
174 174 {
175 175 int error;
176 176
177 177 if ((error = mod_remove(&modlinkage)) != 0) {
178 178 return (error);
179 179 }
180 180 mutex_destroy(&devpoll_lock);
181 181 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
182 182 return (0);
183 183 }
184 184
185 185 int
186 186 _info(struct modinfo *modinfop)
187 187 {
188 188 return (mod_info(&modlinkage, modinfop));
189 189 }
190 190
191 191 /*ARGSUSED*/
192 192 static int
193 193 dpattach(dev_info_t *devi, ddi_attach_cmd_t cmd)
194 194 {
195 195 if (ddi_create_minor_node(devi, "poll", S_IFCHR, 0, DDI_PSEUDO, NULL)
196 196 == DDI_FAILURE) {
197 197 ddi_remove_minor_node(devi, NULL);
198 198 return (DDI_FAILURE);
199 199 }
200 200 dpdevi = devi;
201 201 return (DDI_SUCCESS);
202 202 }
203 203
204 204 static int
205 205 dpdetach(dev_info_t *devi, ddi_detach_cmd_t cmd)
206 206 {
207 207 if (cmd != DDI_DETACH)
208 208 return (DDI_FAILURE);
209 209
210 210 ddi_remove_minor_node(devi, NULL);
211 211 return (DDI_SUCCESS);
212 212 }
213 213
214 214 /* ARGSUSED */
215 215 static int
216 216 dpinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
217 217 {
218 218 int error;
219 219
220 220 switch (infocmd) {
221 221 case DDI_INFO_DEVT2DEVINFO:
222 222 *result = (void *)dpdevi;
223 223 error = DDI_SUCCESS;
224 224 break;
225 225 case DDI_INFO_DEVT2INSTANCE:
226 226 *result = (void *)0;
227 227 error = DDI_SUCCESS;
228 228 break;
229 229 default:
230 230 error = DDI_FAILURE;
231 231 }
232 232 return (error);
233 233 }
234 234
235 235 /*
236 236 * dp_pcache_poll has similar logic to pcache_poll() in poll.c. The major
237 237 * differences are: (1) /dev/poll requires scanning the bitmap starting at
238 238 * where it was stopped last time, instead of always starting from 0,
239 239 * (2) since user may not have cleaned up the cached fds when they are
240 240 * closed, some polldats in cache may refer to closed or reused fds. We
241 241 * need to check for those cases.
242 242 *
243 243 * NOTE: Upon closing an fd, automatic poll cache cleanup is done for
244 244 * poll(2) caches but NOT for /dev/poll caches. So expect some
245 245 * stale entries!
246 246 */
247 247 static int
248 248 dp_pcache_poll(dp_entry_t *dpep, void *dpbuf,
249 249 pollcache_t *pcp, nfds_t nfds, int *fdcntp)
250 250 {
251 251 int start, ostart, end;
252 252 int fdcnt, fd;
253 253 boolean_t done;
254 254 file_t *fp;
255 255 short revent;
256 256 boolean_t no_wrap;
257 257 pollhead_t *php;
258 258 polldat_t *pdp;
259 259 pollfd_t *pfdp;
260 260 epoll_event_t *epoll;
261 261 int error = 0;
262 262 short mask = POLLRDHUP | POLLWRBAND;
263 263 boolean_t is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
264 264
265 265 ASSERT(MUTEX_HELD(&pcp->pc_lock));
266 266 if (pcp->pc_bitmap == NULL) {
267 267 /*
268 268 * No Need to search because no poll fd
269 269 * has been cached.
270 270 */
271 271 return (error);
272 272 }
273 273
274 274 if (is_epoll) {
275 275 pfdp = NULL;
276 276 epoll = (epoll_event_t *)dpbuf;
277 277 } else {
278 278 pfdp = (pollfd_t *)dpbuf;
279 279 epoll = NULL;
280 280 }
281 281 retry:
282 282 start = ostart = pcp->pc_mapstart;
283 283 end = pcp->pc_mapend;
284 284 php = NULL;
285 285
286 286 if (start == 0) {
287 287 /*
288 288 * started from every begining, no need to wrap around.
289 289 */
290 290 no_wrap = B_TRUE;
291 291 } else {
292 292 no_wrap = B_FALSE;
293 293 }
294 294 done = B_FALSE;
295 295 fdcnt = 0;
296 296 while ((fdcnt < nfds) && !done) {
297 297 php = NULL;
298 298 revent = 0;
299 299 /*
300 300 * Examine the bit map in a circular fashion
301 301 * to avoid starvation. Always resume from
302 302 * last stop. Scan till end of the map. Then
303 303 * wrap around.
304 304 */
305 305 fd = bt_getlowbit(pcp->pc_bitmap, start, end);
306 306 ASSERT(fd <= end);
307 307 if (fd >= 0) {
308 308 if (fd == end) {
309 309 if (no_wrap) {
310 310 done = B_TRUE;
311 311 } else {
312 312 start = 0;
313 313 end = ostart - 1;
314 314 no_wrap = B_TRUE;
315 315 }
316 316 } else {
317 317 start = fd + 1;
318 318 }
319 319 pdp = pcache_lookup_fd(pcp, fd);
320 320 repoll:
321 321 ASSERT(pdp != NULL);
322 322 ASSERT(pdp->pd_fd == fd);
323 323 if (pdp->pd_fp == NULL) {
324 324 /*
325 325 * The fd is POLLREMOVed. This fd is
326 326 * logically no longer cached. So move
327 327 * on to the next one.
328 328 */
329 329 continue;
330 330 }
331 331 if ((fp = getf(fd)) == NULL) {
332 332 /*
333 333 * The fd has been closed, but user has not
334 334 * done a POLLREMOVE on this fd yet. Instead
335 335 * of cleaning it here implicitly, we return
336 336 * POLLNVAL. This is consistent with poll(2)
337 337 * polling a closed fd. Hope this will remind
338 338 * user to do a POLLREMOVE.
339 339 */
340 340 if (!is_epoll && pfdp != NULL) {
341 341 pfdp[fdcnt].fd = fd;
342 342 pfdp[fdcnt].revents = POLLNVAL;
343 343 fdcnt++;
344 344 continue;
345 345 }
346 346
347 347 /*
348 348 * In the epoll compatibility case, we actually
349 349 * perform the implicit removal to remain
350 350 * closer to the epoll semantics.
351 351 */
352 352 if (is_epoll) {
353 353 pdp->pd_fp = NULL;
354 354 pdp->pd_events = 0;
355 355
356 356 if (php != NULL) {
357 357 pollhead_delete(php, pdp);
358 358 pdp->pd_php = NULL;
359 359 }
360 360
361 361 BT_CLEAR(pcp->pc_bitmap, fd);
362 362 continue;
363 363 }
364 364 }
365 365
366 366 if (fp != pdp->pd_fp) {
367 367 /*
368 368 * user is polling on a cached fd which was
369 369 * closed and then reused. Unfortunately
370 370 * there is no good way to inform user.
371 371 * If the file struct is also reused, we
372 372 * may not be able to detect the fd reuse
373 373 * at all. As long as this does not
374 374 * cause system failure and/or memory leak,
375 375 * we will play along. Man page states if
376 376 * user does not clean up closed fds, polling
377 377 * results will be indeterministic.
378 378 *
379 379 * XXX - perhaps log the detection of fd
380 380 * reuse?
381 381 */
382 382 pdp->pd_fp = fp;
383 383 }
384 384 /*
385 385 * XXX - pollrelock() logic needs to know which
386 386 * which pollcache lock to grab. It'd be a
387 387 * cleaner solution if we could pass pcp as
388 388 * an arguement in VOP_POLL interface instead
389 389 * of implicitly passing it using thread_t
390 390 * struct. On the other hand, changing VOP_POLL
391 391 * interface will require all driver/file system
392 392 * poll routine to change. May want to revisit
393 393 * the tradeoff later.
394 394 */
395 395 curthread->t_pollcache = pcp;
396 396 error = VOP_POLL(fp->f_vnode, pdp->pd_events, 0,
397 397 &revent, &php, NULL);
398 398 curthread->t_pollcache = NULL;
399 399 releasef(fd);
400 400 if (error != 0) {
401 401 break;
402 402 }
403 403
404 404 /*
405 405 * layered devices (e.g. console driver)
406 406 * may change the vnode and thus the pollhead
407 407 * pointer out from underneath us.
408 408 */
409 409 if (php != NULL && pdp->pd_php != NULL &&
410 410 php != pdp->pd_php) {
411 411 pollhead_delete(pdp->pd_php, pdp);
412 412 pdp->pd_php = php;
413 413 pollhead_insert(php, pdp);
414 414 /*
415 415 * The bit should still be set.
416 416 */
417 417 ASSERT(BT_TEST(pcp->pc_bitmap, fd));
418 418 goto retry;
419 419 }
420 420
421 421 if (revent != 0) {
422 422 if (pfdp != NULL) {
423 423 pfdp[fdcnt].fd = fd;
424 424 pfdp[fdcnt].events = pdp->pd_events;
425 425 pfdp[fdcnt].revents = revent;
426 426 } else if (epoll != NULL) {
427 427 epoll_event_t *ep = &epoll[fdcnt];
428 428
429 429 ASSERT(epoll != NULL);
430 430 ep->data.u64 = pdp->pd_epolldata;
431 431
432 432 /*
433 433 * If any of the event bits are set for
434 434 * which poll and epoll representations
435 435 * differ, swizzle in the native epoll
436 436 * values.
437 437 */
438 438 if (revent & mask) {
439 439 ep->events = (revent & ~mask) |
440 440 ((revent & POLLRDHUP) ?
441 441 EPOLLRDHUP : 0) |
442 442 ((revent & POLLWRBAND) ?
443 443 EPOLLWRBAND : 0);
444 444 } else {
445 445 ep->events = revent;
446 446 }
447 447
448 448 /*
449 449 * We define POLLWRNORM to be POLLOUT,
450 450 * but epoll has separate definitions
451 451 * for them; if POLLOUT is set and the
452 452 * user has asked for EPOLLWRNORM, set
453 453 * that as well.
454 454 */
455 455 if ((revent & POLLOUT) &&
456 456 (pdp->pd_events & EPOLLWRNORM)) {
457 457 ep->events |= EPOLLWRNORM;
458 458 }
459 459 } else {
460 460 pollstate_t *ps =
461 461 curthread->t_pollstate;
462 462 /*
463 463 * The devpoll handle itself is being
464 464 * polled. Notify the caller of any
465 465 * readable event(s), leaving as much
466 466 * state as possible untouched.
467 467 */
468 468 VERIFY(fdcnt == 0);
469 469 VERIFY(ps != NULL);
470 470
471 471 /*
472 472 * If a call to pollunlock() fails
473 473 * during VOP_POLL, skip over the fd
474 474 * and continue polling.
475 475 *
476 476 * Otherwise, report that there is an
477 477 * event pending.
478 478 */
479 479 if ((ps->ps_flags & POLLSTATE_ULFAIL)
480 480 != 0) {
481 481 ps->ps_flags &=
482 482 ~POLLSTATE_ULFAIL;
483 483 continue;
484 484 } else {
485 485 fdcnt++;
486 486 break;
487 487 }
488 488 }
489 489
490 490 /*
491 491 * If POLLET is set, clear the bit in the
492 492 * bitmap -- which effectively latches the
493 493 * edge on a pollwakeup() from the driver.
494 494 */
495 495 if (pdp->pd_events & POLLET)
496 496 BT_CLEAR(pcp->pc_bitmap, fd);
497 497
498 498 /*
499 499 * If POLLONESHOT is set, perform the implicit
500 500 * POLLREMOVE.
501 501 */
502 502 if (pdp->pd_events & POLLONESHOT) {
503 503 pdp->pd_fp = NULL;
504 504 pdp->pd_events = 0;
505 505
506 506 if (php != NULL) {
507 507 pollhead_delete(php, pdp);
508 508 pdp->pd_php = NULL;
509 509 }
510 510
511 511 BT_CLEAR(pcp->pc_bitmap, fd);
512 512 }
513 513
514 514 fdcnt++;
515 515 } else if (php != NULL) {
516 516 /*
517 517 * We clear a bit or cache a poll fd if
518 518 * the driver returns a poll head ptr,
519 519 * which is expected in the case of 0
520 520 * revents. Some buggy driver may return
521 521 * NULL php pointer with 0 revents. In
522 522 * this case, we just treat the driver as
523 523 * "noncachable" and not clearing the bit
524 524 * in bitmap.
525 525 */
526 526 if ((pdp->pd_php != NULL) &&
527 527 ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
528 528 BT_CLEAR(pcp->pc_bitmap, fd);
529 529 }
530 530 if (pdp->pd_php == NULL) {
531 531 pollhead_insert(php, pdp);
532 532 pdp->pd_php = php;
533 533 /*
534 534 * An event of interest may have
535 535 * arrived between the VOP_POLL() and
536 536 * the pollhead_insert(); check again.
537 537 */
538 538 goto repoll;
539 539 }
540 540 }
541 541 } else {
542 542 /*
543 543 * No bit set in the range. Check for wrap around.
544 544 */
545 545 if (!no_wrap) {
546 546 start = 0;
547 547 end = ostart - 1;
548 548 no_wrap = B_TRUE;
549 549 } else {
550 550 done = B_TRUE;
551 551 }
552 552 }
553 553 }
554 554
555 555 if (!done) {
556 556 pcp->pc_mapstart = start;
557 557 }
558 558 ASSERT(*fdcntp == 0);
559 559 *fdcntp = fdcnt;
560 560 return (error);
561 561 }
562 562
563 563 /*ARGSUSED*/
564 564 static int
565 565 dpopen(dev_t *devp, int flag, int otyp, cred_t *credp)
566 566 {
567 567 minor_t minordev;
568 568 dp_entry_t *dpep;
569 569 pollcache_t *pcp;
570 570
571 571 ASSERT(devpoll_init);
572 572 ASSERT(dptblsize <= MAXMIN);
573 573 mutex_enter(&devpoll_lock);
574 574 for (minordev = 0; minordev < dptblsize; minordev++) {
575 575 if (devpolltbl[minordev] == NULL) {
576 576 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
577 577 break;
578 578 }
579 579 }
580 580 if (minordev == dptblsize) {
581 581 dp_entry_t **newtbl;
582 582 size_t oldsize;
583 583
584 584 /*
585 585 * Used up every entry in the existing devpoll table.
586 586 * Grow the table by DEVPOLLSIZE.
587 587 */
588 588 if ((oldsize = dptblsize) >= MAXMIN) {
589 589 mutex_exit(&devpoll_lock);
590 590 return (ENXIO);
591 591 }
592 592 dptblsize += DEVPOLLSIZE;
593 593 if (dptblsize > MAXMIN) {
594 594 dptblsize = MAXMIN;
595 595 }
596 596 newtbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
597 597 bcopy(devpolltbl, newtbl, sizeof (caddr_t) * oldsize);
598 598 kmem_free(devpolltbl, sizeof (caddr_t) * oldsize);
599 599 devpolltbl = newtbl;
600 600 devpolltbl[minordev] = (dp_entry_t *)RESERVED;
601 601 }
602 602 mutex_exit(&devpoll_lock);
603 603
604 604 dpep = kmem_zalloc(sizeof (dp_entry_t), KM_SLEEP);
605 605 /*
606 606 * allocate a pollcache skeleton here. Delay allocating bitmap
607 607 * structures until dpwrite() time, since we don't know the
608 608 * optimal size yet. We also delay setting the pid until either
609 609 * dpwrite() or attempt to poll on the instance, allowing parents
610 610 * to create instances of /dev/poll for their children. (In the
611 611 * epoll compatibility case, this check isn't performed to maintain
612 612 * semantic compatibility.)
613 613 */
614 614 pcp = pcache_alloc();
615 615 dpep->dpe_pcache = pcp;
616 616 pcp->pc_pid = -1;
617 617 *devp = makedevice(getmajor(*devp), minordev); /* clone the driver */
618 618 mutex_enter(&devpoll_lock);
619 619 ASSERT(minordev < dptblsize);
620 620 ASSERT(devpolltbl[minordev] == (dp_entry_t *)RESERVED);
621 621 devpolltbl[minordev] = dpep;
622 622 mutex_exit(&devpoll_lock);
623 623 return (0);
624 624 }
625 625
626 626 /*
627 627 * Write to dev/poll add/remove fd's to/from a cached poll fd set,
628 628 * or change poll events for a watched fd.
629 629 */
630 630 /*ARGSUSED*/
631 631 static int
632 632 dpwrite(dev_t dev, struct uio *uiop, cred_t *credp)
633 633 {
634 634 minor_t minor;
635 635 dp_entry_t *dpep;
636 636 pollcache_t *pcp;
637 637 pollfd_t *pollfdp, *pfdp;
638 638 dvpoll_epollfd_t *epfdp;
639 639 uintptr_t limit;
640 640 int error, size;
641 641 ssize_t uiosize;
642 642 nfds_t pollfdnum;
643 643 struct pollhead *php = NULL;
644 644 polldat_t *pdp;
645 645 int fd;
646 646 file_t *fp;
647 647 boolean_t is_epoll, fds_added = B_FALSE;
648 648
649 649 minor = getminor(dev);
650 650
651 651 mutex_enter(&devpoll_lock);
652 652 ASSERT(minor < dptblsize);
653 653 dpep = devpolltbl[minor];
654 654 ASSERT(dpep != NULL);
655 655 mutex_exit(&devpoll_lock);
656 656
657 657 mutex_enter(&dpep->dpe_lock);
658 658 pcp = dpep->dpe_pcache;
659 659 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
660 660 size = (is_epoll) ? sizeof (dvpoll_epollfd_t) : sizeof (pollfd_t);
661 661 mutex_exit(&dpep->dpe_lock);
662 662
663 663 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
664 664 if (pcp->pc_pid != -1) {
665 665 return (EACCES);
666 666 }
667 667
668 668 pcp->pc_pid = curproc->p_pid;
669 669 }
670 670
671 671 uiosize = uiop->uio_resid;
672 672 pollfdnum = uiosize / size;
673 673 mutex_enter(&curproc->p_lock);
674 674 if (pollfdnum > (uint_t)rctl_enforced_value(
675 675 rctlproc_legacy[RLIMIT_NOFILE], curproc->p_rctls, curproc)) {
676 676 (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
677 677 curproc->p_rctls, curproc, RCA_SAFE);
678 678 mutex_exit(&curproc->p_lock);
679 679 return (EINVAL);
680 680 }
681 681 mutex_exit(&curproc->p_lock);
682 682 /*
683 683 * Copy in the pollfd array. Walk through the array and add
684 684 * each polled fd to the cached set.
685 685 */
686 686 pollfdp = kmem_alloc(uiosize, KM_SLEEP);
687 687 limit = (uintptr_t)pollfdp + (pollfdnum * size);
688 688
689 689 /*
690 690 * Although /dev/poll uses the write(2) interface to cache fds, it's
691 691 * not supposed to function as a seekable device. To prevent offset
692 692 * from growing and eventually exceed the maximum, reset the offset
693 693 * here for every call.
694 694 */
695 695 uiop->uio_loffset = 0;
696 696 if ((error = uiomove((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop))
697 697 != 0) {
698 698 kmem_free(pollfdp, uiosize);
699 699 return (error);
700 700 }
701 701 /*
702 702 * We are about to enter the core portion of dpwrite(). Make sure this
703 703 * write has exclusive access in this portion of the code, i.e., no
704 704 * other writers in this code.
705 705 *
706 706 * Waiting for all readers to drop their references to the dpe is
707 707 * unecessary since the pollcache itself is protected by pc_lock.
708 708 */
709 709 mutex_enter(&dpep->dpe_lock);
710 710 dpep->dpe_writerwait++;
711 711 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0) {
712 712 ASSERT(dpep->dpe_refcnt != 0);
713 713
714 714 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
715 715 dpep->dpe_writerwait--;
716 716 mutex_exit(&dpep->dpe_lock);
717 717 kmem_free(pollfdp, uiosize);
718 718 return (EINTR);
719 719 }
720 720 }
721 721 dpep->dpe_writerwait--;
722 722 dpep->dpe_flag |= DP_WRITER_PRESENT;
723 723 dpep->dpe_refcnt++;
724 724
725 725 if (!is_epoll && (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0) {
726 726 /*
727 727 * The epoll compat mode was enabled while we were waiting to
728 728 * establish write access. It is not safe to continue since
729 729 * state was prepared for non-epoll operation.
730 730 */
731 731 error = EBUSY;
732 732 goto bypass;
733 733 }
734 734 mutex_exit(&dpep->dpe_lock);
735 735
736 736 /*
737 737 * Since the dpwrite() may recursively walk an added /dev/poll handle,
738 738 * pollstate_enter() deadlock and loop detection must be used.
739 739 */
740 740 (void) pollstate_create();
741 741 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
742 742
743 743 if (pcp->pc_bitmap == NULL) {
744 744 pcache_create(pcp, pollfdnum);
745 745 }
746 746 for (pfdp = pollfdp; (uintptr_t)pfdp < limit;
747 747 pfdp = (pollfd_t *)((uintptr_t)pfdp + size)) {
748 748 fd = pfdp->fd;
749 749 if ((uint_t)fd >= P_FINFO(curproc)->fi_nfiles) {
750 750 /*
751 751 * epoll semantics demand that we return EBADF if our
752 752 * specified fd is invalid.
753 753 */
754 754 if (is_epoll) {
755 755 error = EBADF;
756 756 break;
757 757 }
758 758
759 759 continue;
760 760 }
761 761
762 762 pdp = pcache_lookup_fd(pcp, fd);
763 763 if (pfdp->events != POLLREMOVE) {
764 764
765 765 fp = NULL;
766 766
767 767 if (pdp == NULL) {
768 768 /*
769 769 * If we're in epoll compatibility mode, check
770 770 * that the fd is valid before allocating
771 771 * anything for it; epoll semantics demand that
772 772 * we return EBADF if our specified fd is
773 773 * invalid.
774 774 */
775 775 if (is_epoll) {
776 776 if ((fp = getf(fd)) == NULL) {
777 777 error = EBADF;
778 778 break;
779 779 }
780 780 }
781 781
782 782 pdp = pcache_alloc_fd(0);
783 783 pdp->pd_fd = fd;
784 784 pdp->pd_pcache = pcp;
785 785 pcache_insert_fd(pcp, pdp, pollfdnum);
786 786 } else {
787 787 /*
788 788 * epoll semantics demand that we error out if
789 789 * a file descriptor is added twice, which we
790 790 * check (imperfectly) by checking if we both
791 791 * have the file descriptor cached and the
792 792 * file pointer that correponds to the file
793 793 * descriptor matches our cached value. If
794 794 * there is a pointer mismatch, the file
795 795 * descriptor was closed without being removed.
796 796 * The converse is clearly not true, however,
797 797 * so to narrow the window by which a spurious
798 798 * EEXIST may be returned, we also check if
799 799 * this fp has been added to an epoll control
800 800 * descriptor in the past; if it hasn't, we
801 801 * know that this is due to fp reuse -- it's
802 802 * not a true EEXIST case. (By performing this
803 803 * additional check, we limit the window of
804 804 * spurious EEXIST to situations where a single
805 805 * file descriptor is being used across two or
806 806 * more epoll control descriptors -- and even
807 807 * then, the file descriptor must be closed and
808 808 * reused in a relatively tight time span.)
809 809 */
810 810 if (is_epoll) {
811 811 if (pdp->pd_fp != NULL &&
812 812 (fp = getf(fd)) != NULL &&
813 813 fp == pdp->pd_fp &&
814 814 (fp->f_flag2 & FEPOLLED)) {
815 815 error = EEXIST;
816 816 releasef(fd);
817 817 break;
818 818 }
819 819
820 820 /*
821 821 * We have decided that the cached
822 822 * information was stale: it either
823 823 * didn't match, or the fp had never
824 824 * actually been epoll()'d on before.
825 825 * We need to now clear our pd_events
826 826 * to assure that we don't mistakenly
827 827 * operate on cached event disposition.
828 828 */
829 829 pdp->pd_events = 0;
830 830 }
831 831 }
832 832
833 833 if (is_epoll) {
834 834 epfdp = (dvpoll_epollfd_t *)pfdp;
835 835 pdp->pd_epolldata = epfdp->dpep_data;
836 836 }
837 837
838 838 ASSERT(pdp->pd_fd == fd);
839 839 ASSERT(pdp->pd_pcache == pcp);
840 840 if (fd >= pcp->pc_mapsize) {
841 841 mutex_exit(&pcp->pc_lock);
842 842 pcache_grow_map(pcp, fd);
843 843 mutex_enter(&pcp->pc_lock);
844 844 }
845 845 if (fd > pcp->pc_mapend) {
846 846 pcp->pc_mapend = fd;
847 847 }
848 848 if (fp == NULL && (fp = getf(fd)) == NULL) {
849 849 /*
850 850 * The fd is not valid. Since we can't pass
851 851 * this error back in the write() call, set
852 852 * the bit in bitmap to force DP_POLL ioctl
853 853 * to examine it.
854 854 */
855 855 BT_SET(pcp->pc_bitmap, fd);
856 856 pdp->pd_events |= pfdp->events;
857 857 continue;
858 858 }
859 859
860 860 /*
861 861 * To (greatly) reduce EEXIST false positives, we
862 862 * denote that this fp has been epoll()'d. We do this
863 863 * regardless of epoll compatibility mode, as the flag
864 864 * is harmless if not in epoll compatibility mode.
865 865 */
866 866 fp->f_flag2 |= FEPOLLED;
867 867
868 868 /*
869 869 * Don't do VOP_POLL for an already cached fd with
870 870 * same poll events.
871 871 */
872 872 if ((pdp->pd_events == pfdp->events) &&
873 873 (pdp->pd_fp == fp)) {
874 874 /*
875 875 * the events are already cached
876 876 */
877 877 releasef(fd);
878 878 continue;
879 879 }
880 880
881 881 /*
882 882 * do VOP_POLL and cache this poll fd.
883 883 */
884 884 /*
885 885 * XXX - pollrelock() logic needs to know which
886 886 * which pollcache lock to grab. It'd be a
887 887 * cleaner solution if we could pass pcp as
888 888 * an arguement in VOP_POLL interface instead
889 889 * of implicitly passing it using thread_t
890 890 * struct. On the other hand, changing VOP_POLL
891 891 * interface will require all driver/file system
892 892 * poll routine to change. May want to revisit
893 893 * the tradeoff later.
894 894 */
895 895 curthread->t_pollcache = pcp;
896 896 error = VOP_POLL(fp->f_vnode, pfdp->events, 0,
897 897 &pfdp->revents, &php, NULL);
898 898 curthread->t_pollcache = NULL;
899 899 /*
900 900 * We always set the bit when this fd is cached;
901 901 * this forces the first DP_POLL to poll this fd.
902 902 * Real performance gain comes from subsequent
903 903 * DP_POLL. We also attempt a pollhead_insert();
904 904 * if it's not possible, we'll do it in dpioctl().
905 905 */
906 906 BT_SET(pcp->pc_bitmap, fd);
907 907 if (error != 0) {
908 908 releasef(fd);
909 909 break;
910 910 }
911 911 pdp->pd_fp = fp;
912 912 pdp->pd_events |= pfdp->events;
913 913 if (php != NULL) {
914 914 if (pdp->pd_php == NULL) {
915 915 pollhead_insert(php, pdp);
916 916 pdp->pd_php = php;
917 917 } else {
918 918 if (pdp->pd_php != php) {
919 919 pollhead_delete(pdp->pd_php,
920 920 pdp);
921 921 pollhead_insert(php, pdp);
922 922 pdp->pd_php = php;
923 923 }
924 924 }
925 925 }
926 926 fds_added = B_TRUE;
927 927 releasef(fd);
928 928 } else {
929 929 if (pdp == NULL || pdp->pd_fp == NULL) {
930 930 if (is_epoll) {
931 931 /*
932 932 * As with the add case (above), epoll
933 933 * semantics demand that we error out
934 934 * in this case.
935 935 */
936 936 error = ENOENT;
937 937 break;
938 938 }
939 939
940 940 continue;
941 941 }
942 942 ASSERT(pdp->pd_fd == fd);
943 943 pdp->pd_fp = NULL;
944 944 pdp->pd_events = 0;
945 945 ASSERT(pdp->pd_thread == NULL);
946 946 if (pdp->pd_php != NULL) {
947 947 pollhead_delete(pdp->pd_php, pdp);
948 948 pdp->pd_php = NULL;
949 949 }
950 950 BT_CLEAR(pcp->pc_bitmap, fd);
951 951 }
952 952 }
953 953 /*
954 954 * Wake any pollcache waiters so they can check the new descriptors.
955 955 *
956 956 * Any fds added to an recursive-capable pollcache could themselves be
957 957 * /dev/poll handles. To ensure that proper event propagation occurs,
958 958 * parent pollcaches are woken too, so that they can create any needed
959 959 * pollcache links.
960 960 */
961 961 if (fds_added) {
962 962 cv_broadcast(&pcp->pc_cv);
963 963 pcache_wake_parents(pcp);
964 964 }
965 965 pollstate_exit(pcp);
966 966 mutex_enter(&dpep->dpe_lock);
967 967 bypass:
968 968 dpep->dpe_flag &= ~DP_WRITER_PRESENT;
969 969 dpep->dpe_refcnt--;
970 970 cv_broadcast(&dpep->dpe_cv);
971 971 mutex_exit(&dpep->dpe_lock);
972 972 kmem_free(pollfdp, uiosize);
973 973 return (error);
974 974 }
975 975
976 976 #define DP_SIGMASK_RESTORE(ksetp) { \
977 977 if (ksetp != NULL) { \
978 978 mutex_enter(&p->p_lock); \
979 979 if (lwp->lwp_cursig == 0) { \
980 980 t->t_hold = lwp->lwp_sigoldmask; \
981 981 t->t_flag &= ~T_TOMASK; \
982 982 } \
983 983 mutex_exit(&p->p_lock); \
984 984 } \
985 985 }
986 986
987 987 /*ARGSUSED*/
988 988 static int
989 989 dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
990 990 {
991 991 minor_t minor;
992 992 dp_entry_t *dpep;
993 993 pollcache_t *pcp;
994 994 hrtime_t now;
995 995 int error = 0;
996 996 boolean_t is_epoll;
997 997 STRUCT_DECL(dvpoll, dvpoll);
998 998
999 999 if (cmd == DP_POLL || cmd == DP_PPOLL) {
1000 1000 /* do this now, before we sleep on DP_WRITER_PRESENT */
1001 1001 now = gethrtime();
1002 1002 }
1003 1003
1004 1004 minor = getminor(dev);
1005 1005 mutex_enter(&devpoll_lock);
1006 1006 ASSERT(minor < dptblsize);
1007 1007 dpep = devpolltbl[minor];
1008 1008 mutex_exit(&devpoll_lock);
1009 1009 ASSERT(dpep != NULL);
1010 1010 pcp = dpep->dpe_pcache;
1011 1011
1012 1012 mutex_enter(&dpep->dpe_lock);
1013 1013 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
1014 1014
1015 1015 if (cmd == DP_EPOLLCOMPAT) {
1016 1016 if (dpep->dpe_refcnt != 0) {
1017 1017 /*
1018 1018 * We can't turn on epoll compatibility while there
1019 1019 * are outstanding operations.
1020 1020 */
1021 1021 mutex_exit(&dpep->dpe_lock);
1022 1022 return (EBUSY);
1023 1023 }
1024 1024
1025 1025 /*
1026 1026 * epoll compatibility is a one-way street: there's no way
1027 1027 * to turn it off for a particular open.
1028 1028 */
1029 1029 dpep->dpe_flag |= DP_ISEPOLLCOMPAT;
1030 1030 mutex_exit(&dpep->dpe_lock);
1031 1031
1032 1032 return (0);
1033 1033 }
1034 1034
1035 1035 if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
1036 1036 if (pcp->pc_pid != -1) {
1037 1037 mutex_exit(&dpep->dpe_lock);
1038 1038 return (EACCES);
1039 1039 }
1040 1040
1041 1041 pcp->pc_pid = curproc->p_pid;
1042 1042 }
1043 1043
1044 1044 /* Wait until all writers have cleared the handle before continuing */
1045 1045 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0 ||
1046 1046 (dpep->dpe_writerwait != 0)) {
1047 1047 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
1048 1048 mutex_exit(&dpep->dpe_lock);
1049 1049 return (EINTR);
1050 1050 }
1051 1051 }
1052 1052 dpep->dpe_refcnt++;
1053 1053 mutex_exit(&dpep->dpe_lock);
1054 1054
1055 1055 switch (cmd) {
1056 1056 case DP_POLL:
1057 1057 case DP_PPOLL:
1058 1058 {
1059 1059 pollstate_t *ps;
1060 1060 nfds_t nfds;
1061 1061 int fdcnt = 0;
1062 1062 size_t size, fdsize, dpsize;
1063 1063 hrtime_t deadline = 0;
1064 1064 k_sigset_t *ksetp = NULL;
1065 1065 k_sigset_t kset;
1066 1066 sigset_t set;
1067 1067 kthread_t *t = curthread;
1068 1068 klwp_t *lwp = ttolwp(t);
1069 1069 struct proc *p = ttoproc(curthread);
1070 1070
1071 1071 STRUCT_INIT(dvpoll, mode);
1072 1072
1073 1073 /*
1074 1074 * The dp_setp member is only required/consumed for DP_PPOLL,
1075 1075 * which otherwise uses the same structure as DP_POLL.
1076 1076 */
1077 1077 if (cmd == DP_POLL) {
1078 1078 dpsize = (uintptr_t)STRUCT_FADDR(dvpoll, dp_setp) -
1079 1079 (uintptr_t)STRUCT_FADDR(dvpoll, dp_fds);
1080 1080 } else {
1081 1081 ASSERT(cmd == DP_PPOLL);
1082 1082 dpsize = STRUCT_SIZE(dvpoll);
1083 1083 }
1084 1084
1085 1085 if ((mode & FKIOCTL) != 0) {
1086 1086 /* Kernel-internal ioctl call */
1087 1087 bcopy((caddr_t)arg, STRUCT_BUF(dvpoll), dpsize);
1088 1088 error = 0;
1089 1089 } else {
1090 1090 error = copyin((caddr_t)arg, STRUCT_BUF(dvpoll),
1091 1091 dpsize);
1092 1092 }
1093 1093
1094 1094 if (error) {
1095 1095 DP_REFRELE(dpep);
1096 1096 return (EFAULT);
1097 1097 }
1098 1098
1099 1099 deadline = STRUCT_FGET(dvpoll, dp_timeout);
1100 1100 if (deadline > 0) {
1101 1101 /*
1102 1102 * Convert the deadline from relative milliseconds
1103 1103 * to absolute nanoseconds. They must wait for at
1104 1104 * least a tick.
1105 1105 */
1106 1106 deadline = MSEC2NSEC(deadline);
1107 1107 deadline = MAX(deadline, nsec_per_tick);
1108 1108 deadline += now;
1109 1109 }
1110 1110
1111 1111 if (cmd == DP_PPOLL) {
1112 1112 void *setp = STRUCT_FGETP(dvpoll, dp_setp);
1113 1113
1114 1114 if (setp != NULL) {
1115 1115 if (copyin(setp, &set, sizeof (set))) {
1116 1116 DP_REFRELE(dpep);
1117 1117 return (EFAULT);
1118 1118 }
1119 1119
1120 1120 sigutok(&set, &kset);
1121 1121 ksetp = &kset;
1122 1122
1123 1123 mutex_enter(&p->p_lock);
1124 1124 schedctl_finish_sigblock(t);
1125 1125 lwp->lwp_sigoldmask = t->t_hold;
1126 1126 t->t_hold = *ksetp;
1127 1127 t->t_flag |= T_TOMASK;
1128 1128
1129 1129 /*
1130 1130 * Like ppoll() with a non-NULL sigset, we'll
1131 1131 * call cv_reltimedwait_sig() just to check for
1132 1132 * signals. This call will return immediately
1133 1133 * with either 0 (signalled) or -1 (no signal).
1134 1134 * There are some conditions whereby we can
1135 1135 * get 0 from cv_reltimedwait_sig() without
1136 1136 * a true signal (e.g., a directed stop), so
1137 1137 * we restore our signal mask in the unlikely
1138 1138 * event that lwp_cursig is 0.
1139 1139 */
1140 1140 if (!cv_reltimedwait_sig(&t->t_delay_cv,
1141 1141 &p->p_lock, 0, TR_CLOCK_TICK)) {
1142 1142 if (lwp->lwp_cursig == 0) {
1143 1143 t->t_hold = lwp->lwp_sigoldmask;
1144 1144 t->t_flag &= ~T_TOMASK;
1145 1145 }
1146 1146
1147 1147 mutex_exit(&p->p_lock);
1148 1148
1149 1149 DP_REFRELE(dpep);
1150 1150 return (EINTR);
1151 1151 }
1152 1152
1153 1153 mutex_exit(&p->p_lock);
1154 1154 }
1155 1155 }
1156 1156
1157 1157 if ((nfds = STRUCT_FGET(dvpoll, dp_nfds)) == 0) {
1158 1158 /*
1159 1159 * We are just using DP_POLL to sleep, so
1160 1160 * we don't any of the devpoll apparatus.
1161 1161 * Do not check for signals if we have a zero timeout.
1162 1162 */
1163 1163 DP_REFRELE(dpep);
1164 1164 if (deadline == 0) {
1165 1165 DP_SIGMASK_RESTORE(ksetp);
1166 1166 return (0);
1167 1167 }
1168 1168
1169 1169 mutex_enter(&curthread->t_delay_lock);
1170 1170 while ((error =
1171 1171 cv_timedwait_sig_hrtime(&curthread->t_delay_cv,
1172 1172 &curthread->t_delay_lock, deadline)) > 0)
1173 1173 continue;
1174 1174 mutex_exit(&curthread->t_delay_lock);
1175 1175
1176 1176 DP_SIGMASK_RESTORE(ksetp);
1177 1177
1178 1178 return (error == 0 ? EINTR : 0);
1179 1179 }
1180 1180
1181 1181 if (is_epoll) {
1182 1182 size = nfds * (fdsize = sizeof (epoll_event_t));
1183 1183 } else {
1184 1184 size = nfds * (fdsize = sizeof (pollfd_t));
1185 1185 }
1186 1186
1187 1187 /*
1188 1188 * XXX It would be nice not to have to alloc each time, but it
1189 1189 * requires another per thread structure hook. This can be
1190 1190 * implemented later if data suggests that it's necessary.
1191 1191 */
1192 1192 ps = pollstate_create();
1193 1193
1194 1194 if (ps->ps_dpbufsize < size) {
1195 1195 /*
1196 1196 * If nfds is larger than twice the current maximum
1197 1197 * open file count, we'll silently clamp it. This
1198 1198 * only limits our exposure to allocating an
1199 1199 * inordinate amount of kernel memory; it doesn't
1200 1200 * otherwise affect the semantics. (We have this
1201 1201 * check at twice the maximum instead of merely the
1202 1202 * maximum because some applications pass an nfds that
1203 1203 * is only slightly larger than their limit.)
1204 1204 */
1205 1205 mutex_enter(&p->p_lock);
1206 1206 if ((nfds >> 1) > p->p_fno_ctl) {
1207 1207 nfds = p->p_fno_ctl;
1208 1208 size = nfds * fdsize;
1209 1209 }
1210 1210 mutex_exit(&p->p_lock);
1211 1211
1212 1212 if (ps->ps_dpbufsize < size) {
1213 1213 kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
1214 1214 ps->ps_dpbuf = kmem_zalloc(size, KM_SLEEP);
1215 1215 ps->ps_dpbufsize = size;
1216 1216 }
1217 1217 }
1218 1218
1219 1219 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
1220 1220 for (;;) {
1221 1221 pcp->pc_flag &= ~PC_POLLWAKE;
1222 1222
1223 1223 /*
1224 1224 * Mark all child pcachelinks as stale.
1225 1225 * Those which are still part of the tree will be
1226 1226 * marked as valid during the poll.
1227 1227 */
1228 1228 pcachelink_mark_stale(pcp);
1229 1229
1230 1230 error = dp_pcache_poll(dpep, ps->ps_dpbuf,
1231 1231 pcp, nfds, &fdcnt);
1232 1232 if (fdcnt > 0 || error != 0)
1233 1233 break;
1234 1234
1235 1235 /* Purge still-stale child pcachelinks */
1236 1236 pcachelink_purge_stale(pcp);
1237 1237
1238 1238 /*
1239 1239 * A pollwake has happened since we polled cache.
1240 1240 */
1241 1241 if (pcp->pc_flag & PC_POLLWAKE)
1242 1242 continue;
1243 1243
1244 1244 /*
1245 1245 * Sleep until we are notified, signaled, or timed out.
1246 1246 */
1247 1247 if (deadline == 0) {
1248 1248 /* immediate timeout; do not check signals */
1249 1249 break;
1250 1250 }
1251 1251
1252 1252 error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
1253 1253 &pcp->pc_lock, deadline);
1254 1254
1255 1255 /*
1256 1256 * If we were awakened by a signal or timeout then
1257 1257 * break the loop, else poll again.
1258 1258 */
1259 1259 if (error <= 0) {
1260 1260 error = (error == 0) ? EINTR : 0;
1261 1261 break;
1262 1262 } else {
1263 1263 error = 0;
1264 1264 }
1265 1265 }
1266 1266 pollstate_exit(pcp);
1267 1267
1268 1268 DP_SIGMASK_RESTORE(ksetp);
1269 1269
1270 1270 if (error == 0 && fdcnt > 0) {
1271 1271 if (copyout(ps->ps_dpbuf,
1272 1272 STRUCT_FGETP(dvpoll, dp_fds), fdcnt * fdsize)) {
1273 1273 DP_REFRELE(dpep);
1274 1274 return (EFAULT);
1275 1275 }
1276 1276 *rvalp = fdcnt;
1277 1277 }
1278 1278 break;
1279 1279 }
1280 1280
1281 1281 case DP_ISPOLLED:
1282 1282 {
1283 1283 pollfd_t pollfd;
1284 1284 polldat_t *pdp;
1285 1285
1286 1286 STRUCT_INIT(dvpoll, mode);
1287 1287 error = copyin((caddr_t)arg, &pollfd, sizeof (pollfd_t));
1288 1288 if (error) {
1289 1289 DP_REFRELE(dpep);
1290 1290 return (EFAULT);
1291 1291 }
1292 1292 mutex_enter(&pcp->pc_lock);
1293 1293 if (pcp->pc_hash == NULL) {
1294 1294 /*
1295 1295 * No Need to search because no poll fd
1296 1296 * has been cached.
1297 1297 */
1298 1298 mutex_exit(&pcp->pc_lock);
1299 1299 DP_REFRELE(dpep);
1300 1300 return (0);
1301 1301 }
1302 1302 if (pollfd.fd < 0) {
1303 1303 mutex_exit(&pcp->pc_lock);
1304 1304 break;
1305 1305 }
1306 1306 pdp = pcache_lookup_fd(pcp, pollfd.fd);
1307 1307 if ((pdp != NULL) && (pdp->pd_fd == pollfd.fd) &&
1308 1308 (pdp->pd_fp != NULL)) {
1309 1309 pollfd.revents = pdp->pd_events;
1310 1310 if (copyout(&pollfd, (caddr_t)arg, sizeof (pollfd_t))) {
1311 1311 mutex_exit(&pcp->pc_lock);
1312 1312 DP_REFRELE(dpep);
1313 1313 return (EFAULT);
1314 1314 }
1315 1315 *rvalp = 1;
1316 1316 }
1317 1317 mutex_exit(&pcp->pc_lock);
1318 1318 break;
1319 1319 }
1320 1320
1321 1321 default:
1322 1322 DP_REFRELE(dpep);
1323 1323 return (EINVAL);
1324 1324 }
1325 1325 DP_REFRELE(dpep);
1326 1326 return (error);
1327 1327 }
1328 1328
1329 1329 /*
1330 1330 * Overview of Recursive Polling
1331 1331 *
1332 1332 * It is possible for /dev/poll to poll for events on file descriptors which
1333 1333 * themselves are /dev/poll handles. Pending events in the child handle are
1334 1334 * represented as readable data via the POLLIN flag. To limit surface area,
1335 1335 * this recursion is presently allowed on only /dev/poll handles which have
1336 1336 * been placed in epoll mode via the DP_EPOLLCOMPAT ioctl. Recursion depth is
1337 1337 * limited to 5 in order to be consistent with Linux epoll.
1338 1338 *
1339 1339 * Extending dppoll() for VOP_POLL:
1340 1340 *
1341 1341 * The recursive /dev/poll implementation begins by extending dppoll() to
1342 1342 * report when resources contained in the pollcache have relevant event state.
1343 1343 * At the highest level, it means calling dp_pcache_poll() so it indicates if
1344 1344 * fd events are present without consuming them or altering the pollcache
1345 1345 * bitmap. This ensures that a subsequent DP_POLL operation on the bitmap will
1346 1346 * yield the initiating event. Additionally, the VOP_POLL should return in
1347 1347 * such a way that dp_pcache_poll() does not clear the parent bitmap entry
1348 1348 * which corresponds to the child /dev/poll fd. This means that child
1349 1349 * pollcaches will be checked during every poll which facilitates wake-up
1350 1350 * behavior detailed below.
1351 1351 *
1352 1352 * Pollcache Links and Wake Events:
1353 1353 *
1354 1354 * Recursive /dev/poll avoids complicated pollcache locking constraints during
1355 1355 * pollwakeup events by eschewing the traditional pollhead mechanism in favor
1356 1356 * of a different approach. For each pollcache at the root of a recursive
1357 1357 * /dev/poll "tree", pcachelink_t structures are established to all child
1358 1358 * /dev/poll pollcaches. During pollnotify() in a child pollcache, the
1359 1359 * linked list of pcachelink_t entries is walked, where those marked as valid
1360 1360 * incur a cv_broadcast to their parent pollcache. Most notably, these
1361 1361 * pcachelink_t cv wakeups are performed without acquiring pc_lock on the
1362 1362 * parent pollcache (which would require careful deadlock avoidance). This
1363 1363 * still allows the woken poll on the parent to discover the pertinent events
1364 1364 * due to the fact that bitmap entires for the child pollcache are always
1365 1365 * maintained by the dppoll() logic above.
1366 1366 *
1367 1367 * Depth Limiting and Loop Prevention:
1368 1368 *
1369 1369 * As each pollcache is encountered (either via DP_POLL or dppoll()), depth and
1370 1370 * loop constraints are enforced via pollstate_enter(). The pollcache_t
1371 1371 * pointer is compared against any existing entries in ps_pc_stack and is added
1372 1372 * to the end if no match (and therefore loop) is found. Once poll operations
1373 1373 * for a given pollcache_t are complete, pollstate_exit() clears the pointer
1374 1374 * from the list. The pollstate_enter() and pollstate_exit() functions are
1375 1375 * responsible for acquiring and releasing pc_lock, respectively.
1376 1376 *
1377 1377 * Deadlock Safety:
1378 1378 *
1379 1379 * Descending through a tree of recursive /dev/poll handles involves the tricky
1380 1380 * business of sequentially entering multiple pollcache locks. This tree
1381 1381 * topology cannot define a lock acquisition order in such a way that it is
1382 1382 * immune to deadlocks between threads. The pollstate_enter() and
1383 1383 * pollstate_exit() functions provide an interface for recursive /dev/poll
1384 1384 * operations to safely lock pollcaches while failing gracefully in the face of
1385 1385 * deadlocking topologies. (See pollstate_contend() for more detail about how
1386 1386 * deadlocks are detected and resolved.)
1387 1387 */
1388 1388
1389 1389 /*ARGSUSED*/
1390 1390 static int
1391 1391 dppoll(dev_t dev, short events, int anyyet, short *reventsp,
1392 1392 struct pollhead **phpp)
1393 1393 {
1394 1394 minor_t minor;
1395 1395 dp_entry_t *dpep;
1396 1396 pollcache_t *pcp;
1397 1397 int res, rc = 0;
1398 1398
1399 1399 minor = getminor(dev);
1400 1400 mutex_enter(&devpoll_lock);
1401 1401 ASSERT(minor < dptblsize);
1402 1402 dpep = devpolltbl[minor];
1403 1403 ASSERT(dpep != NULL);
1404 1404 mutex_exit(&devpoll_lock);
1405 1405
1406 1406 mutex_enter(&dpep->dpe_lock);
1407 1407 if ((dpep->dpe_flag & DP_ISEPOLLCOMPAT) == 0) {
1408 1408 /* Poll recursion is not yet supported for non-epoll handles */
1409 1409 *reventsp = POLLERR;
1410 1410 mutex_exit(&dpep->dpe_lock);
1411 1411 return (0);
1412 1412 } else {
1413 1413 dpep->dpe_refcnt++;
1414 1414 pcp = dpep->dpe_pcache;
1415 1415 mutex_exit(&dpep->dpe_lock);
1416 1416 }
1417 1417
1418 1418 res = pollstate_enter(pcp);
1419 1419 if (res == PSE_SUCCESS) {
1420 1420 nfds_t nfds = 1;
1421 1421 int fdcnt = 0;
1422 1422 pollstate_t *ps = curthread->t_pollstate;
1423 1423
1424 1424 rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt);
1425 1425 if (rc == 0) {
1426 1426 *reventsp = (fdcnt > 0) ? POLLIN : 0;
1427 1427 }
1428 1428 pcachelink_assoc(pcp, ps->ps_pc_stack[0]);
1429 1429 pollstate_exit(pcp);
1430 1430 } else {
1431 1431 switch (res) {
1432 1432 case PSE_FAIL_DEPTH:
1433 1433 rc = EINVAL;
1434 1434 break;
1435 1435 case PSE_FAIL_LOOP:
1436 1436 case PSE_FAIL_DEADLOCK:
1437 1437 rc = ELOOP;
1438 1438 break;
1439 1439 default:
1440 1440 /*
1441 1441 * If anything else has gone awry, such as being polled
1442 1442 * from an unexpected context, fall back to the
1443 1443 * recursion-intolerant response.
1444 1444 */
1445 1445 *reventsp = POLLERR;
1446 1446 rc = 0;
1447 1447 break;
1448 1448 }
1449 1449 }
1450 1450
1451 1451 DP_REFRELE(dpep);
1452 1452 return (rc);
1453 1453 }
1454 1454
1455 1455 /*
1456 1456 * devpoll close should do enough clean up before the pollcache is deleted,
1457 1457 * i.e., it should ensure no one still references the pollcache later.
1458 1458 * There is no "permission" check in here. Any process having the last
1459 1459 * reference of this /dev/poll fd can close.
1460 1460 */
1461 1461 /*ARGSUSED*/
1462 1462 static int
1463 1463 dpclose(dev_t dev, int flag, int otyp, cred_t *credp)
1464 1464 {
1465 1465 minor_t minor;
1466 1466 dp_entry_t *dpep;
1467 1467 pollcache_t *pcp;
1468 1468 int i;
1469 1469 polldat_t **hashtbl;
1470 1470 polldat_t *pdp;
1471 1471
1472 1472 minor = getminor(dev);
1473 1473
1474 1474 mutex_enter(&devpoll_lock);
1475 1475 dpep = devpolltbl[minor];
1476 1476 ASSERT(dpep != NULL);
1477 1477 devpolltbl[minor] = NULL;
1478 1478 mutex_exit(&devpoll_lock);
1479 1479 pcp = dpep->dpe_pcache;
1480 1480 ASSERT(pcp != NULL);
1481 1481 /*
1482 1482 * At this point, no other lwp can access this pollcache via the
1483 1483 * /dev/poll fd. This pollcache is going away, so do the clean
1484 1484 * up without the pc_lock.
1485 1485 */
1486 1486 hashtbl = pcp->pc_hash;
1487 1487 for (i = 0; i < pcp->pc_hashsize; i++) {
1488 1488 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
1489 1489 if (pdp->pd_php != NULL) {
1490 1490 pollhead_delete(pdp->pd_php, pdp);
1491 1491 pdp->pd_php = NULL;
1492 1492 pdp->pd_fp = NULL;
1493 1493 }
1494 1494 }
1495 1495 }
1496 1496 /*
1497 1497 * pollwakeup() may still interact with this pollcache. Wait until
1498 1498 * it is done.
1499 1499 */
1500 1500 mutex_enter(&pcp->pc_no_exit);
1501 1501 ASSERT(pcp->pc_busy >= 0);
1502 1502 while (pcp->pc_busy > 0)
1503 1503 cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
1504 1504 mutex_exit(&pcp->pc_no_exit);
1505 1505
1506 1506 /* Clean up any pollcache links created via recursive /dev/poll */
1507 1507 if (pcp->pc_parents != NULL || pcp->pc_children != NULL) {
1508 1508 /*
1509 1509 * Because of the locking rules for pcachelink manipulation,
1510 1510 * acquring pc_lock is required for this step.
1511 1511 */
1512 1512 mutex_enter(&pcp->pc_lock);
1513 1513 pcachelink_purge_all(pcp);
1514 1514 mutex_exit(&pcp->pc_lock);
1515 1515 }
1516 1516
1517 1517 pcache_destroy(pcp);
1518 1518 ASSERT(dpep->dpe_refcnt == 0);
1519 1519 kmem_free(dpep, sizeof (dp_entry_t));
1520 1520 return (0);
1521 1521 }
1522 1522
1523 1523 static void
1524 1524 pcachelink_locked_rele(pcachelink_t *pl)
1525 1525 {
1526 1526 ASSERT(MUTEX_HELD(&pl->pcl_lock));
1527 1527 VERIFY(pl->pcl_refcnt >= 1);
1528 1528
1529 1529 pl->pcl_refcnt--;
1530 1530 if (pl->pcl_refcnt == 0) {
1531 1531 VERIFY(pl->pcl_state == PCL_INVALID);
1532 1532 ASSERT(pl->pcl_parent_pc == NULL);
1533 1533 ASSERT(pl->pcl_child_pc == NULL);
1534 1534 ASSERT(pl->pcl_parent_next == NULL);
1535 1535 ASSERT(pl->pcl_child_next == NULL);
1536 1536
1537 1537 pl->pcl_state = PCL_FREE;
1538 1538 mutex_destroy(&pl->pcl_lock);
1539 1539 kmem_free(pl, sizeof (pcachelink_t));
1540 1540 } else {
1541 1541 mutex_exit(&pl->pcl_lock);
1542 1542 }
1543 1543 }
1544 1544
1545 1545 /*
1546 1546 * Associate parent and child pollcaches via a pcachelink_t. If an existing
1547 1547 * link (stale or valid) between the two is found, it will be reused. If a
1548 1548 * suitable link is not found for reuse, a new one will be allocated.
1549 1549 */
1550 1550 static void
1551 1551 pcachelink_assoc(pollcache_t *child, pollcache_t *parent)
1552 1552 {
1553 1553 pcachelink_t *pl, **plpn;
1554 1554
1555 1555 ASSERT(MUTEX_HELD(&child->pc_lock));
1556 1556 ASSERT(MUTEX_HELD(&parent->pc_lock));
1557 1557
1558 1558 /* Search for an existing link we can reuse. */
1559 1559 plpn = &child->pc_parents;
1560 1560 for (pl = child->pc_parents; pl != NULL; pl = *plpn) {
1561 1561 mutex_enter(&pl->pcl_lock);
1562 1562 if (pl->pcl_state == PCL_INVALID) {
1563 1563 /* Clean any invalid links while walking the list */
1564 1564 *plpn = pl->pcl_parent_next;
1565 1565 pl->pcl_child_pc = NULL;
1566 1566 pl->pcl_parent_next = NULL;
1567 1567 pcachelink_locked_rele(pl);
1568 1568 } else if (pl->pcl_parent_pc == parent) {
1569 1569 /* Successfully found parent link */
1570 1570 ASSERT(pl->pcl_state == PCL_VALID ||
1571 1571 pl->pcl_state == PCL_STALE);
1572 1572 pl->pcl_state = PCL_VALID;
1573 1573 mutex_exit(&pl->pcl_lock);
1574 1574 return;
1575 1575 } else {
1576 1576 plpn = &pl->pcl_parent_next;
1577 1577 mutex_exit(&pl->pcl_lock);
1578 1578 }
1579 1579 }
1580 1580
1581 1581 /* No existing link to the parent was found. Create a fresh one. */
1582 1582 pl = kmem_zalloc(sizeof (pcachelink_t), KM_SLEEP);
1583 1583 mutex_init(&pl->pcl_lock, NULL, MUTEX_DEFAULT, NULL);
1584 1584
1585 1585 pl->pcl_parent_pc = parent;
1586 1586 pl->pcl_child_next = parent->pc_children;
1587 1587 parent->pc_children = pl;
1588 1588 pl->pcl_refcnt++;
1589 1589
1590 1590 pl->pcl_child_pc = child;
1591 1591 pl->pcl_parent_next = child->pc_parents;
1592 1592 child->pc_parents = pl;
1593 1593 pl->pcl_refcnt++;
1594 1594
1595 1595 pl->pcl_state = PCL_VALID;
1596 1596 }
1597 1597
1598 1598 /*
1599 1599 * Mark all child links in a pollcache as stale. Any invalid child links found
1600 1600 * during iteration are purged.
1601 1601 */
1602 1602 static void
1603 1603 pcachelink_mark_stale(pollcache_t *pcp)
1604 1604 {
1605 1605 pcachelink_t *pl, **plpn;
1606 1606
1607 1607 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1608 1608
1609 1609 plpn = &pcp->pc_children;
1610 1610 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1611 1611 mutex_enter(&pl->pcl_lock);
1612 1612 if (pl->pcl_state == PCL_INVALID) {
1613 1613 /*
1614 1614 * Remove any invalid links while we are going to the
1615 1615 * trouble of walking the list.
1616 1616 */
1617 1617 *plpn = pl->pcl_child_next;
1618 1618 pl->pcl_parent_pc = NULL;
1619 1619 pl->pcl_child_next = NULL;
1620 1620 pcachelink_locked_rele(pl);
1621 1621 } else {
1622 1622 pl->pcl_state = PCL_STALE;
1623 1623 plpn = &pl->pcl_child_next;
1624 1624 mutex_exit(&pl->pcl_lock);
1625 1625 }
1626 1626 }
1627 1627 }
1628 1628
1629 1629 /*
1630 1630 * Purge all stale (or invalid) child links from a pollcache.
1631 1631 */
1632 1632 static void
1633 1633 pcachelink_purge_stale(pollcache_t *pcp)
1634 1634 {
1635 1635 pcachelink_t *pl, **plpn;
1636 1636
1637 1637 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1638 1638
1639 1639 plpn = &pcp->pc_children;
1640 1640 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1641 1641 mutex_enter(&pl->pcl_lock);
1642 1642 switch (pl->pcl_state) {
1643 1643 case PCL_STALE:
1644 1644 pl->pcl_state = PCL_INVALID;
1645 1645 /* FALLTHROUGH */
1646 1646 case PCL_INVALID:
1647 1647 *plpn = pl->pcl_child_next;
1648 1648 pl->pcl_parent_pc = NULL;
1649 1649 pl->pcl_child_next = NULL;
1650 1650 pcachelink_locked_rele(pl);
1651 1651 break;
1652 1652 default:
1653 1653 plpn = &pl->pcl_child_next;
1654 1654 mutex_exit(&pl->pcl_lock);
1655 1655 }
1656 1656 }
1657 1657 }
1658 1658
1659 1659 /*
1660 1660 * Purge all child and parent links from a pollcache, regardless of status.
1661 1661 */
1662 1662 static void
1663 1663 pcachelink_purge_all(pollcache_t *pcp)
1664 1664 {
1665 1665 pcachelink_t *pl, **plpn;
1666 1666
1667 1667 ASSERT(MUTEX_HELD(&pcp->pc_lock));
1668 1668
1669 1669 plpn = &pcp->pc_parents;
1670 1670 for (pl = pcp->pc_parents; pl != NULL; pl = *plpn) {
1671 1671 mutex_enter(&pl->pcl_lock);
1672 1672 pl->pcl_state = PCL_INVALID;
1673 1673 *plpn = pl->pcl_parent_next;
1674 1674 pl->pcl_child_pc = NULL;
1675 1675 pl->pcl_parent_next = NULL;
1676 1676 pcachelink_locked_rele(pl);
1677 1677 }
1678 1678
1679 1679 plpn = &pcp->pc_children;
1680 1680 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1681 1681 mutex_enter(&pl->pcl_lock);
1682 1682 pl->pcl_state = PCL_INVALID;
1683 1683 *plpn = pl->pcl_child_next;
1684 1684 pl->pcl_parent_pc = NULL;
1685 1685 pl->pcl_child_next = NULL;
1686 1686 pcachelink_locked_rele(pl);
1687 1687 }
1688 1688
1689 1689 ASSERT(pcp->pc_parents == NULL);
1690 1690 ASSERT(pcp->pc_children == NULL);
1691 1691 }
|
↓ open down ↓ |
1558 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX