1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /* Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved. */
27
28 #include <sys/types.h>
29 #include <sys/t_lock.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/buf.h>
33 #include <sys/conf.h>
34 #include <sys/cred.h>
35 #include <sys/kmem.h>
36 #include <sys/sysmacros.h>
37 #include <sys/vfs.h>
38 #include <sys/vnode.h>
39 #include <sys/debug.h>
40 #include <sys/errno.h>
41 #include <sys/time.h>
42 #include <sys/file.h>
43 #include <sys/user.h>
44 #include <sys/stream.h>
45 #include <sys/strsubr.h>
46 #include <sys/strsun.h>
47 #include <sys/sunddi.h>
48 #include <sys/esunddi.h>
49 #include <sys/flock.h>
50 #include <sys/modctl.h>
51 #include <sys/cmn_err.h>
52 #include <sys/vmsystm.h>
53 #include <sys/policy.h>
54
55 #include <sys/socket.h>
56 #include <sys/socketvar.h>
57
58 #include <sys/isa_defs.h>
59 #include <sys/inttypes.h>
60 #include <sys/systm.h>
61 #include <sys/cpuvar.h>
62 #include <sys/filio.h>
63 #include <sys/sendfile.h>
64 #include <sys/ddi.h>
65 #include <vm/seg.h>
66 #include <vm/seg_map.h>
67 #include <vm/seg_kpm.h>
68
69 #include <fs/sockfs/nl7c.h>
70 #include <fs/sockfs/sockcommon.h>
71 #include <fs/sockfs/sockfilter_impl.h>
72 #include <fs/sockfs/socktpi.h>
73
74 #ifdef SOCK_TEST
75 int do_useracc = 1; /* Controlled by setting SO_DEBUG to 4 */
76 #else
77 #define do_useracc 1
78 #endif /* SOCK_TEST */
79
80 extern int xnet_truncate_print;
81
82 extern void nl7c_init(void);
83 extern int sockfs_defer_nl7c_init;
84
85 /*
86 * Note: DEF_IOV_MAX is defined and used as it is in "fs/vncalls.c"
87 * as there isn't a formal definition of IOV_MAX ???
88 */
89 #define MSG_MAXIOVLEN 16
90
91 /*
92 * Kernel component of socket creation.
93 *
94 * The socket library determines which version number to use.
95 * First the library calls this with a NULL devpath. If this fails
96 * to find a transport (using solookup) the library will look in /etc/netconfig
97 * for the appropriate transport. If one is found it will pass in the
98 * devpath for the kernel to use.
99 */
100 int
101 so_socket(int family, int type_w_flags, int protocol, char *devpath,
102 int version)
103 {
104 struct sonode *so;
105 vnode_t *vp;
106 struct file *fp;
107 int fd;
108 int error;
109 int type;
110
111 type = type_w_flags & SOCK_TYPE_MASK;
112 type_w_flags &= ~SOCK_TYPE_MASK;
113 if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK))
114 return (set_errno(EINVAL));
115
116 if (devpath != NULL) {
117 char *buf;
118 size_t kdevpathlen = 0;
119
120 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
121 if ((error = copyinstr(devpath, buf,
122 MAXPATHLEN, &kdevpathlen)) != 0) {
123 kmem_free(buf, MAXPATHLEN);
124 return (set_errno(error));
125 }
126 so = socket_create(family, type, protocol, buf, NULL,
127 SOCKET_SLEEP, version, CRED(), &error);
128 kmem_free(buf, MAXPATHLEN);
129 } else {
130 so = socket_create(family, type, protocol, NULL, NULL,
131 SOCKET_SLEEP, version, CRED(), &error);
132 }
133 if (so == NULL)
134 return (set_errno(error));
135
136 /* Allocate a file descriptor for the socket */
137 vp = SOTOV(so);
138 if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) {
139 (void) socket_close(so, 0, CRED());
140 socket_destroy(so);
141 return (set_errno(error));
142 }
143
144 /*
145 * Now fill in the entries that falloc reserved
146 */
147 if (type_w_flags & SOCK_NDELAY) {
148 so->so_state |= SS_NDELAY;
149 fp->f_flag |= FNDELAY;
150 }
151 if (type_w_flags & SOCK_NONBLOCK) {
152 so->so_state |= SS_NONBLOCK;
153 fp->f_flag |= FNONBLOCK;
154 }
155 mutex_exit(&fp->f_tlock);
156 setf(fd, fp);
157 if ((type_w_flags & SOCK_CLOEXEC) != 0) {
158 f_setfd(fd, FD_CLOEXEC);
159 }
160
161 return (fd);
162 }
163
164 /*
165 * Map from a file descriptor to a socket node.
166 * Returns with the file descriptor held i.e. the caller has to
167 * use releasef when done with the file descriptor.
168 */
169 struct sonode *
170 getsonode(int sock, int *errorp, file_t **fpp)
171 {
172 file_t *fp;
173 vnode_t *vp;
174 struct sonode *so;
175
176 if ((fp = getf(sock)) == NULL) {
177 *errorp = EBADF;
178 eprintline(*errorp);
179 return (NULL);
180 }
181 vp = fp->f_vnode;
182 /* Check if it is a socket */
183 if (vp->v_type != VSOCK) {
184 releasef(sock);
185 *errorp = ENOTSOCK;
186 eprintline(*errorp);
187 return (NULL);
188 }
189 /*
190 * Use the stream head to find the real socket vnode.
191 * This is needed when namefs sits above sockfs.
192 */
193 if (vp->v_stream) {
194 ASSERT(vp->v_stream->sd_vnode);
195 vp = vp->v_stream->sd_vnode;
196
197 so = VTOSO(vp);
198 if (so->so_version == SOV_STREAM) {
199 releasef(sock);
200 *errorp = ENOTSOCK;
201 eprintsoline(so, *errorp);
202 return (NULL);
203 }
204 } else {
205 so = VTOSO(vp);
206 }
207 if (fpp)
208 *fpp = fp;
209 return (so);
210 }
211
212 /*
213 * Allocate and copyin a sockaddr.
214 * Ensures NULL termination for AF_UNIX addresses by extending them
215 * with one NULL byte if need be. Verifies that the length is not
216 * excessive to prevent an application from consuming all of kernel
217 * memory. Returns NULL when an error occurred.
218 */
219 static struct sockaddr *
220 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp,
221 int *errorp)
222 {
223 char *faddr;
224 size_t namelen = (size_t)*namelenp;
225
226 ASSERT(namelen != 0);
227 if (namelen > SO_MAXARGSIZE) {
228 *errorp = EINVAL;
229 eprintsoline(so, *errorp);
230 return (NULL);
231 }
232
233 faddr = (char *)kmem_alloc(namelen, KM_SLEEP);
234 if (copyin(name, faddr, namelen)) {
235 kmem_free(faddr, namelen);
236 *errorp = EFAULT;
237 eprintsoline(so, *errorp);
238 return (NULL);
239 }
240
241 /*
242 * Add space for NULL termination if needed.
243 * Do a quick check if the last byte is NUL.
244 */
245 if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') {
246 /* Check if there is any NULL termination */
247 size_t i;
248 int foundnull = 0;
249
250 for (i = sizeof (name->sa_family); i < namelen; i++) {
251 if (faddr[i] == '\0') {
252 foundnull = 1;
253 break;
254 }
255 }
256 if (!foundnull) {
257 /* Add extra byte for NUL padding */
258 char *nfaddr;
259
260 nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP);
261 bcopy(faddr, nfaddr, namelen);
262 kmem_free(faddr, namelen);
263
264 /* NUL terminate */
265 nfaddr[namelen] = '\0';
266 namelen++;
267 ASSERT((socklen_t)namelen == namelen);
268 *namelenp = (socklen_t)namelen;
269 faddr = nfaddr;
270 }
271 }
272 return ((struct sockaddr *)faddr);
273 }
274
275 /*
276 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
277 */
278 static int
279 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp,
280 void *kaddr, socklen_t klen)
281 {
282 if (uaddr != NULL) {
283 if (ulen > klen)
284 ulen = klen;
285
286 if (ulen != 0) {
287 if (copyout(kaddr, uaddr, ulen))
288 return (EFAULT);
289 }
290 } else
291 ulen = 0;
292
293 if (ulenp != NULL) {
294 if (copyout(&ulen, ulenp, sizeof (ulen)))
295 return (EFAULT);
296 }
297 return (0);
298 }
299
300 /*
301 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
302 * If klen is greater than ulen it still uses the non-truncated
303 * klen to update ulenp.
304 */
305 static int
306 copyout_name(void *uaddr, socklen_t ulen, void *ulenp,
307 void *kaddr, socklen_t klen)
308 {
309 if (uaddr != NULL) {
310 if (ulen >= klen)
311 ulen = klen;
312 else if (ulen != 0 && xnet_truncate_print) {
313 printf("sockfs: truncating copyout of address using "
314 "XNET semantics for pid = %d. Lengths %d, %d\n",
315 curproc->p_pid, klen, ulen);
316 }
317
318 if (ulen != 0) {
319 if (copyout(kaddr, uaddr, ulen))
320 return (EFAULT);
321 } else
322 klen = 0;
323 } else
324 klen = 0;
325
326 if (ulenp != NULL) {
327 if (copyout(&klen, ulenp, sizeof (klen)))
328 return (EFAULT);
329 }
330 return (0);
331 }
332
333 /*
334 * The socketpair() code in libsocket creates two sockets (using
335 * the /etc/netconfig fallback if needed) before calling this routine
336 * to connect the two sockets together.
337 *
338 * For a SOCK_STREAM socketpair a listener is needed - in that case this
339 * routine will create a new file descriptor as part of accepting the
340 * connection. The library socketpair() will check if svs[2] has changed
341 * in which case it will close the changed fd.
342 *
343 * Note that this code could use the TPI feature of accepting the connection
344 * on the listening endpoint. However, that would require significant changes
345 * to soaccept.
346 */
347 int
348 so_socketpair(int sv[2])
349 {
350 int svs[2];
351 struct sonode *so1, *so2;
352 int error;
353 int orig_flags;
354 struct sockaddr_ux *name;
355 size_t namelen;
356 sotpi_info_t *sti1;
357 sotpi_info_t *sti2;
358
359 dprint(1, ("so_socketpair(%p)\n", (void *)sv));
360
361 error = useracc(sv, sizeof (svs), B_WRITE);
362 if (error && do_useracc)
363 return (set_errno(EFAULT));
364
365 if (copyin(sv, svs, sizeof (svs)))
366 return (set_errno(EFAULT));
367
368 if ((so1 = getsonode(svs[0], &error, NULL)) == NULL)
369 return (set_errno(error));
370
371 if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) {
372 releasef(svs[0]);
373 return (set_errno(error));
374 }
375
376 if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) {
377 error = EOPNOTSUPP;
378 goto done;
379 }
380
381 sti1 = SOTOTPI(so1);
382 sti2 = SOTOTPI(so2);
383
384 /*
385 * The code below makes assumptions about the "sockfs" implementation.
386 * So make sure that the correct implementation is really used.
387 */
388 ASSERT(so1->so_ops == &sotpi_sonodeops);
389 ASSERT(so2->so_ops == &sotpi_sonodeops);
390
391 if (so1->so_type == SOCK_DGRAM) {
392 /*
393 * Bind both sockets and connect them with each other.
394 * Need to allocate name/namelen for soconnect.
395 */
396 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED());
397 if (error) {
398 eprintsoline(so1, error);
399 goto done;
400 }
401 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
402 if (error) {
403 eprintsoline(so2, error);
404 goto done;
405 }
406 namelen = sizeof (struct sockaddr_ux);
407 name = kmem_alloc(namelen, KM_SLEEP);
408 name->sou_family = AF_UNIX;
409 name->sou_addr = sti2->sti_ux_laddr;
410 error = socket_connect(so1,
411 (struct sockaddr *)name,
412 (socklen_t)namelen,
413 0, _SOCONNECT_NOXLATE, CRED());
414 if (error) {
415 kmem_free(name, namelen);
416 eprintsoline(so1, error);
417 goto done;
418 }
419 name->sou_addr = sti1->sti_ux_laddr;
420 error = socket_connect(so2,
421 (struct sockaddr *)name,
422 (socklen_t)namelen,
423 0, _SOCONNECT_NOXLATE, CRED());
424 kmem_free(name, namelen);
425 if (error) {
426 eprintsoline(so2, error);
427 goto done;
428 }
429 releasef(svs[0]);
430 releasef(svs[1]);
431 } else {
432 /*
433 * Bind both sockets, with so1 being a listener.
434 * Connect so2 to so1 - nonblocking to avoid waiting for
435 * soaccept to complete.
436 * Accept a connection on so1. Pass out the new fd as sv[0].
437 * The library will detect the changed fd and close
438 * the original one.
439 */
440 struct sonode *nso;
441 struct vnode *nvp;
442 struct file *nfp;
443 int nfd;
444
445 /*
446 * We could simply call socket_listen() here (which would do the
447 * binding automatically) if the code didn't rely on passing
448 * _SOBIND_NOXLATE to the TPI implementation of socket_bind().
449 */
450 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC|
451 _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR,
452 CRED());
453 if (error) {
454 eprintsoline(so1, error);
455 goto done;
456 }
457 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
458 if (error) {
459 eprintsoline(so2, error);
460 goto done;
461 }
462
463 namelen = sizeof (struct sockaddr_ux);
464 name = kmem_alloc(namelen, KM_SLEEP);
465 name->sou_family = AF_UNIX;
466 name->sou_addr = sti1->sti_ux_laddr;
467 error = socket_connect(so2,
468 (struct sockaddr *)name,
469 (socklen_t)namelen,
470 FNONBLOCK, _SOCONNECT_NOXLATE, CRED());
471 kmem_free(name, namelen);
472 if (error) {
473 if (error != EINPROGRESS) {
474 eprintsoline(so2, error); goto done;
475 }
476 }
477
478 error = socket_accept(so1, 0, CRED(), &nso);
479 if (error) {
480 eprintsoline(so1, error);
481 goto done;
482 }
483
484 /* wait for so2 being SS_CONNECTED ignoring signals */
485 mutex_enter(&so2->so_lock);
486 error = sowaitconnected(so2, 0, 1);
487 mutex_exit(&so2->so_lock);
488 if (error != 0) {
489 (void) socket_close(nso, 0, CRED());
490 socket_destroy(nso);
491 eprintsoline(so2, error);
492 goto done;
493 }
494
495 nvp = SOTOV(nso);
496 if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) {
497 (void) socket_close(nso, 0, CRED());
498 socket_destroy(nso);
499 eprintsoline(nso, error);
500 goto done;
501 }
502 /*
503 * copy over FNONBLOCK and FNDELAY flags should they exist
504 */
505 if (so1->so_state & SS_NONBLOCK)
506 nfp->f_flag |= FNONBLOCK;
507 if (so1->so_state & SS_NDELAY)
508 nfp->f_flag |= FNDELAY;
509
510 /*
511 * fill in the entries that falloc reserved
512 */
513 mutex_exit(&nfp->f_tlock);
514 setf(nfd, nfp);
515
516 /*
517 * get the original flags before we release
518 */
519 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0);
520
521 releasef(svs[0]);
522 releasef(svs[1]);
523
524 /*
525 * If FD_CLOEXEC was set on the filedescriptor we're
526 * swapping out, we should set it on the new one too.
527 */
528 if (orig_flags & FD_CLOEXEC) {
529 f_setfd(nfd, FD_CLOEXEC);
530 }
531
532 /*
533 * The socketpair library routine will close the original
534 * svs[0] when this code passes out a different file
535 * descriptor.
536 */
537 svs[0] = nfd;
538
539 if (copyout(svs, sv, sizeof (svs))) {
540 (void) closeandsetf(nfd, NULL);
541 eprintline(EFAULT);
542 return (set_errno(EFAULT));
543 }
544 }
545 return (0);
546
547 done:
548 releasef(svs[0]);
549 releasef(svs[1]);
550 return (set_errno(error));
551 }
552
553 int
554 bind(int sock, struct sockaddr *name, socklen_t namelen, int version)
555 {
556 struct sonode *so;
557 int error;
558
559 dprint(1, ("bind(%d, %p, %d)\n",
560 sock, (void *)name, namelen));
561
562 if ((so = getsonode(sock, &error, NULL)) == NULL)
563 return (set_errno(error));
564
565 /* Allocate and copyin name */
566 /*
567 * X/Open test does not expect EFAULT with NULL name and non-zero
568 * namelen.
569 */
570 if (name != NULL && namelen != 0) {
571 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
572 name = copyin_name(so, name, &namelen, &error);
573 if (name == NULL) {
574 releasef(sock);
575 return (set_errno(error));
576 }
577 } else {
578 name = NULL;
579 namelen = 0;
580 }
581
582 switch (version) {
583 default:
584 error = socket_bind(so, name, namelen, 0, CRED());
585 break;
586 case SOV_XPG4_2:
587 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED());
588 break;
589 case SOV_SOCKBSD:
590 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED());
591 break;
592 }
593 done:
594 releasef(sock);
595 if (name != NULL)
596 kmem_free(name, (size_t)namelen);
597
598 if (error)
599 return (set_errno(error));
600 return (0);
601 }
602
603 /* ARGSUSED2 */
604 int
605 listen(int sock, int backlog, int version)
606 {
607 struct sonode *so;
608 int error;
609
610 dprint(1, ("listen(%d, %d)\n",
611 sock, backlog));
612
613 if ((so = getsonode(sock, &error, NULL)) == NULL)
614 return (set_errno(error));
615
616 error = socket_listen(so, backlog, CRED());
617
618 releasef(sock);
619 if (error)
620 return (set_errno(error));
621 return (0);
622 }
623
624 /*ARGSUSED3*/
625 int
626 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version,
627 int flags)
628 {
629 struct sonode *so;
630 file_t *fp;
631 int error;
632 socklen_t namelen;
633 struct sonode *nso;
634 struct vnode *nvp;
635 struct file *nfp;
636 int nfd;
637 int ssflags;
638 struct sockaddr *addrp;
639 socklen_t addrlen;
640
641 dprint(1, ("accept(%d, %p, %p)\n",
642 sock, (void *)name, (void *)namelenp));
643
644 if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) {
645 return (set_errno(EINVAL));
646 }
647
648 /* Translate SOCK_ flags to their SS_ variant */
649 ssflags = 0;
650 if (flags & SOCK_NONBLOCK)
651 ssflags |= SS_NONBLOCK;
652 if (flags & SOCK_NDELAY)
653 ssflags |= SS_NDELAY;
654
655 if ((so = getsonode(sock, &error, &fp)) == NULL)
656 return (set_errno(error));
657
658 if (name != NULL) {
659 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
660 if (copyin(namelenp, &namelen, sizeof (namelen))) {
661 releasef(sock);
662 return (set_errno(EFAULT));
663 }
664 if (namelen != 0) {
665 error = useracc(name, (size_t)namelen, B_WRITE);
666 if (error && do_useracc) {
667 releasef(sock);
668 return (set_errno(EFAULT));
669 }
670 } else
671 name = NULL;
672 } else {
673 namelen = 0;
674 }
675
676 /*
677 * Allocate the user fd before socket_accept() in order to
678 * catch EMFILE errors before calling socket_accept().
679 */
680 if ((nfd = ufalloc(0)) == -1) {
681 eprintsoline(so, EMFILE);
682 releasef(sock);
683 return (set_errno(EMFILE));
684 }
685 error = socket_accept(so, fp->f_flag, CRED(), &nso);
686 if (error) {
687 setf(nfd, NULL);
688 releasef(sock);
689 return (set_errno(error));
690 }
691
692 nvp = SOTOV(nso);
693
694 ASSERT(MUTEX_NOT_HELD(&nso->so_lock));
695 if (namelen != 0) {
696 addrlen = so->so_max_addr_len;
697 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP);
698
699 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp,
700 &addrlen, B_TRUE, CRED())) == 0) {
701 error = copyout_name(name, namelen, namelenp,
702 addrp, addrlen);
703 } else {
704 ASSERT(error == EINVAL || error == ENOTCONN);
705 error = ECONNABORTED;
706 }
707 kmem_free(addrp, so->so_max_addr_len);
708 }
709
710 if (error) {
711 setf(nfd, NULL);
712 (void) socket_close(nso, 0, CRED());
713 socket_destroy(nso);
714 releasef(sock);
715 return (set_errno(error));
716 }
717 if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) {
718 setf(nfd, NULL);
719 (void) socket_close(nso, 0, CRED());
720 socket_destroy(nso);
721 eprintsoline(so, error);
722 releasef(sock);
723 return (set_errno(error));
724 }
725 /*
726 * fill in the entries that falloc reserved
727 */
728 nfp->f_vnode = nvp;
729 mutex_exit(&nfp->f_tlock);
730 setf(nfd, nfp);
731
732 /*
733 * Act on SOCK_CLOEXEC from flags
734 */
735 if (flags & SOCK_CLOEXEC) {
736 f_setfd(nfd, FD_CLOEXEC);
737 }
738
739 /*
740 * Copy FNDELAY and FNONBLOCK from listener to acceptor
741 * and from ssflags
742 */
743 if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) {
744 uint_t oflag = nfp->f_flag;
745 int arg = 0;
746
747 if ((ssflags | so->so_state) & SS_NONBLOCK)
748 arg |= FNONBLOCK;
749 else if ((ssflags | so->so_state) & SS_NDELAY)
750 arg |= FNDELAY;
751
752 /*
753 * This code is a simplification of the F_SETFL code in fcntl()
754 * Ignore any errors from VOP_SETFL.
755 */
756 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL))
757 != 0) {
758 eprintsoline(so, error);
759 error = 0;
760 } else {
761 mutex_enter(&nfp->f_tlock);
762 nfp->f_flag &= ~FMASK | (FREAD|FWRITE);
763 nfp->f_flag |= arg;
764 mutex_exit(&nfp->f_tlock);
765 }
766 }
767 releasef(sock);
768 return (nfd);
769 }
770
771 int
772 connect(int sock, struct sockaddr *name, socklen_t namelen, int version)
773 {
774 struct sonode *so;
775 file_t *fp;
776 int error;
777
778 dprint(1, ("connect(%d, %p, %d)\n",
779 sock, (void *)name, namelen));
780
781 if ((so = getsonode(sock, &error, &fp)) == NULL)
782 return (set_errno(error));
783
784 /* Allocate and copyin name */
785 if (namelen != 0) {
786 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
787 name = copyin_name(so, name, &namelen, &error);
788 if (name == NULL) {
789 releasef(sock);
790 return (set_errno(error));
791 }
792 } else
793 name = NULL;
794
795 error = socket_connect(so, name, namelen, fp->f_flag,
796 (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED());
797 releasef(sock);
798 if (name)
799 kmem_free(name, (size_t)namelen);
800 if (error)
801 return (set_errno(error));
802 return (0);
803 }
804
805 /*ARGSUSED2*/
806 int
807 shutdown(int sock, int how, int version)
808 {
809 struct sonode *so;
810 int error;
811
812 dprint(1, ("shutdown(%d, %d)\n",
813 sock, how));
814
815 if ((so = getsonode(sock, &error, NULL)) == NULL)
816 return (set_errno(error));
817
818 error = socket_shutdown(so, how, CRED());
819
820 releasef(sock);
821 if (error)
822 return (set_errno(error));
823 return (0);
824 }
825
826 /*
827 * Common receive routine.
828 */
829 static ssize_t
830 recvit(int sock,
831 struct nmsghdr *msg,
832 struct uio *uiop,
833 int flags,
834 socklen_t *namelenp,
835 socklen_t *controllenp,
836 int *flagsp)
837 {
838 struct sonode *so;
839 file_t *fp;
840 void *name;
841 socklen_t namelen;
842 void *control;
843 socklen_t controllen;
844 ssize_t len;
845 int error;
846
847 if ((so = getsonode(sock, &error, &fp)) == NULL)
848 return (set_errno(error));
849
850 len = uiop->uio_resid;
851 uiop->uio_fmode = fp->f_flag;
852 uiop->uio_extflg = UIO_COPY_CACHED;
853
854 name = msg->msg_name;
855 namelen = msg->msg_namelen;
856 control = msg->msg_control;
857 controllen = msg->msg_controllen;
858
859 msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL |
860 MSG_DONTWAIT | MSG_XPG4_2);
861
862 error = socket_recvmsg(so, msg, uiop, CRED());
863 if (error) {
864 releasef(sock);
865 return (set_errno(error));
866 }
867 lwp_stat_update(LWP_STAT_MSGRCV, 1);
868 releasef(sock);
869
870 error = copyout_name(name, namelen, namelenp,
871 msg->msg_name, msg->msg_namelen);
872 if (error)
873 goto err;
874
875 if (flagsp != NULL) {
876 /*
877 * Clear internal flag.
878 */
879 msg->msg_flags &= ~MSG_XPG4_2;
880
881 /*
882 * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only
883 * when controllen is zero and there is control data to
884 * copy out.
885 */
886 if (controllen != 0 &&
887 (msg->msg_controllen > controllen || control == NULL)) {
888 dprint(1, ("recvit: CTRUNC %d %d %p\n",
889 msg->msg_controllen, controllen, control));
890
891 msg->msg_flags |= MSG_CTRUNC;
892 }
893 if (copyout(&msg->msg_flags, flagsp,
894 sizeof (msg->msg_flags))) {
895 error = EFAULT;
896 goto err;
897 }
898 }
899 /*
900 * Note: This MUST be done last. There can be no "goto err" after this
901 * point since it could make so_closefds run twice on some part
902 * of the file descriptor array.
903 */
904 if (controllen != 0) {
905 if (!(flags & MSG_XPG4_2)) {
906 /*
907 * Good old msg_accrights can only return a multiple
908 * of 4 bytes.
909 */
910 controllen &= ~((int)sizeof (uint32_t) - 1);
911 }
912 error = copyout_arg(control, controllen, controllenp,
913 msg->msg_control, msg->msg_controllen);
914 if (error)
915 goto err;
916
917 if (msg->msg_controllen > controllen || control == NULL) {
918 if (control == NULL)
919 controllen = 0;
920 so_closefds(msg->msg_control, msg->msg_controllen,
921 !(flags & MSG_XPG4_2), controllen);
922 }
923 }
924 if (msg->msg_namelen != 0)
925 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
926 if (msg->msg_controllen != 0)
927 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
928 return (len - uiop->uio_resid);
929
930 err:
931 /*
932 * If we fail and the control part contains file descriptors
933 * we have to close the fd's.
934 */
935 if (msg->msg_controllen != 0)
936 so_closefds(msg->msg_control, msg->msg_controllen,
937 !(flags & MSG_XPG4_2), 0);
938 if (msg->msg_namelen != 0)
939 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
940 if (msg->msg_controllen != 0)
941 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
942 return (set_errno(error));
943 }
944
945 /*
946 * Native system call
947 */
948 ssize_t
949 recv(int sock, void *buffer, size_t len, int flags)
950 {
951 struct nmsghdr lmsg;
952 struct uio auio;
953 struct iovec aiov[1];
954
955 dprint(1, ("recv(%d, %p, %ld, %d)\n",
956 sock, buffer, len, flags));
957
958 if ((ssize_t)len < 0) {
959 return (set_errno(EINVAL));
960 }
961
962 aiov[0].iov_base = buffer;
963 aiov[0].iov_len = len;
964 auio.uio_loffset = 0;
965 auio.uio_iov = aiov;
966 auio.uio_iovcnt = 1;
967 auio.uio_resid = len;
968 auio.uio_segflg = UIO_USERSPACE;
969 auio.uio_limit = 0;
970
971 lmsg.msg_namelen = 0;
972 lmsg.msg_controllen = 0;
973 lmsg.msg_flags = 0;
974 return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL));
975 }
976
977 ssize_t
978 recvfrom(int sock, void *buffer, size_t len, int flags,
979 struct sockaddr *name, socklen_t *namelenp)
980 {
981 struct nmsghdr lmsg;
982 struct uio auio;
983 struct iovec aiov[1];
984
985 dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n",
986 sock, buffer, len, flags, (void *)name, (void *)namelenp));
987
988 if ((ssize_t)len < 0) {
989 return (set_errno(EINVAL));
990 }
991
992 aiov[0].iov_base = buffer;
993 aiov[0].iov_len = len;
994 auio.uio_loffset = 0;
995 auio.uio_iov = aiov;
996 auio.uio_iovcnt = 1;
997 auio.uio_resid = len;
998 auio.uio_segflg = UIO_USERSPACE;
999 auio.uio_limit = 0;
1000
1001 lmsg.msg_name = (char *)name;
1002 if (namelenp != NULL) {
1003 if (copyin(namelenp, &lmsg.msg_namelen,
1004 sizeof (lmsg.msg_namelen)))
1005 return (set_errno(EFAULT));
1006 } else {
1007 lmsg.msg_namelen = 0;
1008 }
1009 lmsg.msg_controllen = 0;
1010 lmsg.msg_flags = 0;
1011
1012 return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL));
1013 }
1014
1015 /*
1016 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1017 * struct omsghdr or struct nmsghdr.
1018 */
1019 ssize_t
1020 recvmsg(int sock, struct nmsghdr *msg, int flags)
1021 {
1022 STRUCT_DECL(nmsghdr, u_lmsg);
1023 STRUCT_HANDLE(nmsghdr, umsgptr);
1024 struct nmsghdr lmsg;
1025 struct uio auio;
1026 struct iovec aiov[MSG_MAXIOVLEN];
1027 int iovcnt;
1028 ssize_t len;
1029 int i;
1030 int *flagsp;
1031 model_t model;
1032
1033 dprint(1, ("recvmsg(%d, %p, %d)\n",
1034 sock, (void *)msg, flags));
1035
1036 model = get_udatamodel();
1037 STRUCT_INIT(u_lmsg, model);
1038 STRUCT_SET_HANDLE(umsgptr, model, msg);
1039
1040 if (flags & MSG_XPG4_2) {
1041 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg)))
1042 return (set_errno(EFAULT));
1043 flagsp = STRUCT_FADDR(umsgptr, msg_flags);
1044 } else {
1045 /*
1046 * Assumes that nmsghdr and omsghdr are identically shaped
1047 * except for the added msg_flags field.
1048 */
1049 if (copyin(msg, STRUCT_BUF(u_lmsg),
1050 SIZEOF_STRUCT(omsghdr, model)))
1051 return (set_errno(EFAULT));
1052 STRUCT_FSET(u_lmsg, msg_flags, 0);
1053 flagsp = NULL;
1054 }
1055
1056 /*
1057 * Code below us will kmem_alloc memory and hang it
1058 * off msg_control and msg_name fields. This forces
1059 * us to copy the structure to its native form.
1060 */
1061 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1062 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1063 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1064 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1065 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1066 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1067 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1068
1069 iovcnt = lmsg.msg_iovlen;
1070
1071 if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) {
1072 return (set_errno(EMSGSIZE));
1073 }
1074
1075 #ifdef _SYSCALL32_IMPL
1076 /*
1077 * 32-bit callers need to have their iovec expanded, while ensuring
1078 * that they can't move more than 2Gbytes of data in a single call.
1079 */
1080 if (model == DATAMODEL_ILP32) {
1081 struct iovec32 aiov32[MSG_MAXIOVLEN];
1082 ssize32_t count32;
1083
1084 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32,
1085 iovcnt * sizeof (struct iovec32)))
1086 return (set_errno(EFAULT));
1087
1088 count32 = 0;
1089 for (i = 0; i < iovcnt; i++) {
1090 ssize32_t iovlen32;
1091
1092 iovlen32 = aiov32[i].iov_len;
1093 count32 += iovlen32;
1094 if (iovlen32 < 0 || count32 < 0)
1095 return (set_errno(EINVAL));
1096 aiov[i].iov_len = iovlen32;
1097 aiov[i].iov_base =
1098 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1099 }
1100 } else
1101 #endif /* _SYSCALL32_IMPL */
1102 if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) {
1103 return (set_errno(EFAULT));
1104 }
1105 len = 0;
1106 for (i = 0; i < iovcnt; i++) {
1107 ssize_t iovlen = aiov[i].iov_len;
1108 len += iovlen;
1109 if (iovlen < 0 || len < 0) {
1110 return (set_errno(EINVAL));
1111 }
1112 }
1113 auio.uio_loffset = 0;
1114 auio.uio_iov = aiov;
1115 auio.uio_iovcnt = iovcnt;
1116 auio.uio_resid = len;
1117 auio.uio_segflg = UIO_USERSPACE;
1118 auio.uio_limit = 0;
1119
1120 if (lmsg.msg_control != NULL &&
1121 (do_useracc == 0 ||
1122 useracc(lmsg.msg_control, lmsg.msg_controllen,
1123 B_WRITE) != 0)) {
1124 return (set_errno(EFAULT));
1125 }
1126
1127 return (recvit(sock, &lmsg, &auio, flags,
1128 STRUCT_FADDR(umsgptr, msg_namelen),
1129 STRUCT_FADDR(umsgptr, msg_controllen), flagsp));
1130 }
1131
1132 /*
1133 * Common send function.
1134 */
1135 static ssize_t
1136 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags)
1137 {
1138 struct sonode *so;
1139 file_t *fp;
1140 void *name;
1141 socklen_t namelen;
1142 void *control;
1143 socklen_t controllen;
1144 ssize_t len;
1145 int error;
1146
1147 if ((so = getsonode(sock, &error, &fp)) == NULL)
1148 return (set_errno(error));
1149
1150 uiop->uio_fmode = fp->f_flag;
1151
1152 if (so->so_family == AF_UNIX)
1153 uiop->uio_extflg = UIO_COPY_CACHED;
1154 else
1155 uiop->uio_extflg = UIO_COPY_DEFAULT;
1156
1157 /* Allocate and copyin name and control */
1158 name = msg->msg_name;
1159 namelen = msg->msg_namelen;
1160 if (name != NULL && namelen != 0) {
1161 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1162 name = copyin_name(so,
1163 (struct sockaddr *)name,
1164 &namelen, &error);
1165 if (name == NULL)
1166 goto done3;
1167 /* copyin_name null terminates addresses for AF_UNIX */
1168 msg->msg_namelen = namelen;
1169 msg->msg_name = name;
1170 } else {
1171 msg->msg_name = name = NULL;
1172 msg->msg_namelen = namelen = 0;
1173 }
1174
1175 control = msg->msg_control;
1176 controllen = msg->msg_controllen;
1177 if ((control != NULL) && (controllen != 0)) {
1178 /*
1179 * Verify that the length is not excessive to prevent
1180 * an application from consuming all of kernel memory.
1181 */
1182 if (controllen > SO_MAXARGSIZE) {
1183 error = EINVAL;
1184 goto done2;
1185 }
1186 control = kmem_alloc(controllen, KM_SLEEP);
1187
1188 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1189 if (copyin(msg->msg_control, control, controllen)) {
1190 error = EFAULT;
1191 goto done1;
1192 }
1193 msg->msg_control = control;
1194 } else {
1195 msg->msg_control = control = NULL;
1196 msg->msg_controllen = controllen = 0;
1197 }
1198
1199 len = uiop->uio_resid;
1200 msg->msg_flags = flags;
1201
1202 error = socket_sendmsg(so, msg, uiop, CRED());
1203 done1:
1204 if (control != NULL)
1205 kmem_free(control, controllen);
1206 done2:
1207 if (name != NULL)
1208 kmem_free(name, namelen);
1209 done3:
1210 if (error != 0) {
1211 releasef(sock);
1212 return (set_errno(error));
1213 }
1214 lwp_stat_update(LWP_STAT_MSGSND, 1);
1215 releasef(sock);
1216 return (len - uiop->uio_resid);
1217 }
1218
1219 /*
1220 * Native system call
1221 */
1222 ssize_t
1223 send(int sock, void *buffer, size_t len, int flags)
1224 {
1225 struct nmsghdr lmsg;
1226 struct uio auio;
1227 struct iovec aiov[1];
1228
1229 dprint(1, ("send(%d, %p, %ld, %d)\n",
1230 sock, buffer, len, flags));
1231
1232 if ((ssize_t)len < 0) {
1233 return (set_errno(EINVAL));
1234 }
1235
1236 aiov[0].iov_base = buffer;
1237 aiov[0].iov_len = len;
1238 auio.uio_loffset = 0;
1239 auio.uio_iov = aiov;
1240 auio.uio_iovcnt = 1;
1241 auio.uio_resid = len;
1242 auio.uio_segflg = UIO_USERSPACE;
1243 auio.uio_limit = 0;
1244
1245 lmsg.msg_name = NULL;
1246 lmsg.msg_control = NULL;
1247 if (!(flags & MSG_XPG4_2)) {
1248 /*
1249 * In order to be compatible with the libsocket/sockmod
1250 * implementation we set EOR for all send* calls.
1251 */
1252 flags |= MSG_EOR;
1253 }
1254 return (sendit(sock, &lmsg, &auio, flags));
1255 }
1256
1257 /*
1258 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1259 * struct omsghdr or struct nmsghdr.
1260 */
1261 ssize_t
1262 sendmsg(int sock, struct nmsghdr *msg, int flags)
1263 {
1264 struct nmsghdr lmsg;
1265 STRUCT_DECL(nmsghdr, u_lmsg);
1266 struct uio auio;
1267 struct iovec aiov[MSG_MAXIOVLEN];
1268 int iovcnt;
1269 ssize_t len;
1270 int i;
1271 model_t model;
1272
1273 dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags));
1274
1275 model = get_udatamodel();
1276 STRUCT_INIT(u_lmsg, model);
1277
1278 if (flags & MSG_XPG4_2) {
1279 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1280 STRUCT_SIZE(u_lmsg)))
1281 return (set_errno(EFAULT));
1282 } else {
1283 /*
1284 * Assumes that nmsghdr and omsghdr are identically shaped
1285 * except for the added msg_flags field.
1286 */
1287 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1288 SIZEOF_STRUCT(omsghdr, model)))
1289 return (set_errno(EFAULT));
1290 /*
1291 * In order to be compatible with the libsocket/sockmod
1292 * implementation we set EOR for all send* calls.
1293 */
1294 flags |= MSG_EOR;
1295 }
1296
1297 /*
1298 * Code below us will kmem_alloc memory and hang it
1299 * off msg_control and msg_name fields. This forces
1300 * us to copy the structure to its native form.
1301 */
1302 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1303 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1304 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1305 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1306 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1307 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1308 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1309
1310 iovcnt = lmsg.msg_iovlen;
1311
1312 if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) {
1313 /*
1314 * Unless this is XPG 4.2 we allow iovcnt == 0 to
1315 * be compatible with SunOS 4.X and 4.4BSD.
1316 */
1317 if (iovcnt != 0 || (flags & MSG_XPG4_2))
1318 return (set_errno(EMSGSIZE));
1319 }
1320
1321 #ifdef _SYSCALL32_IMPL
1322 /*
1323 * 32-bit callers need to have their iovec expanded, while ensuring
1324 * that they can't move more than 2Gbytes of data in a single call.
1325 */
1326 if (model == DATAMODEL_ILP32) {
1327 struct iovec32 aiov32[MSG_MAXIOVLEN];
1328 ssize32_t count32;
1329
1330 if (iovcnt != 0 &&
1331 copyin((struct iovec32 *)lmsg.msg_iov, aiov32,
1332 iovcnt * sizeof (struct iovec32)))
1333 return (set_errno(EFAULT));
1334
1335 count32 = 0;
1336 for (i = 0; i < iovcnt; i++) {
1337 ssize32_t iovlen32;
1338
1339 iovlen32 = aiov32[i].iov_len;
1340 count32 += iovlen32;
1341 if (iovlen32 < 0 || count32 < 0)
1342 return (set_errno(EINVAL));
1343 aiov[i].iov_len = iovlen32;
1344 aiov[i].iov_base =
1345 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1346 }
1347 } else
1348 #endif /* _SYSCALL32_IMPL */
1349 if (iovcnt != 0 &&
1350 copyin(lmsg.msg_iov, aiov,
1351 (unsigned)iovcnt * sizeof (struct iovec))) {
1352 return (set_errno(EFAULT));
1353 }
1354 len = 0;
1355 for (i = 0; i < iovcnt; i++) {
1356 ssize_t iovlen = aiov[i].iov_len;
1357 len += iovlen;
1358 if (iovlen < 0 || len < 0) {
1359 return (set_errno(EINVAL));
1360 }
1361 }
1362 auio.uio_loffset = 0;
1363 auio.uio_iov = aiov;
1364 auio.uio_iovcnt = iovcnt;
1365 auio.uio_resid = len;
1366 auio.uio_segflg = UIO_USERSPACE;
1367 auio.uio_limit = 0;
1368
1369 return (sendit(sock, &lmsg, &auio, flags));
1370 }
1371
1372 ssize_t
1373 sendto(int sock, void *buffer, size_t len, int flags,
1374 struct sockaddr *name, socklen_t namelen)
1375 {
1376 struct nmsghdr lmsg;
1377 struct uio auio;
1378 struct iovec aiov[1];
1379
1380 dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n",
1381 sock, buffer, len, flags, (void *)name, namelen));
1382
1383 if ((ssize_t)len < 0) {
1384 return (set_errno(EINVAL));
1385 }
1386
1387 aiov[0].iov_base = buffer;
1388 aiov[0].iov_len = len;
1389 auio.uio_loffset = 0;
1390 auio.uio_iov = aiov;
1391 auio.uio_iovcnt = 1;
1392 auio.uio_resid = len;
1393 auio.uio_segflg = UIO_USERSPACE;
1394 auio.uio_limit = 0;
1395
1396 lmsg.msg_name = (char *)name;
1397 lmsg.msg_namelen = namelen;
1398 lmsg.msg_control = NULL;
1399 if (!(flags & MSG_XPG4_2)) {
1400 /*
1401 * In order to be compatible with the libsocket/sockmod
1402 * implementation we set EOR for all send* calls.
1403 */
1404 flags |= MSG_EOR;
1405 }
1406 return (sendit(sock, &lmsg, &auio, flags));
1407 }
1408
1409 /*ARGSUSED3*/
1410 int
1411 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1412 {
1413 struct sonode *so;
1414 int error;
1415 socklen_t namelen;
1416 socklen_t sock_addrlen;
1417 struct sockaddr *sock_addrp;
1418
1419 dprint(1, ("getpeername(%d, %p, %p)\n",
1420 sock, (void *)name, (void *)namelenp));
1421
1422 if ((so = getsonode(sock, &error, NULL)) == NULL)
1423 goto bad;
1424
1425 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1426 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1427 (name == NULL && namelen != 0)) {
1428 error = EFAULT;
1429 goto rel_out;
1430 }
1431 sock_addrlen = so->so_max_addr_len;
1432 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1433
1434 if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen,
1435 B_FALSE, CRED())) == 0) {
1436 ASSERT(sock_addrlen <= so->so_max_addr_len);
1437 error = copyout_name(name, namelen, namelenp,
1438 (void *)sock_addrp, sock_addrlen);
1439 }
1440 kmem_free(sock_addrp, so->so_max_addr_len);
1441 rel_out:
1442 releasef(sock);
1443 bad: return (error != 0 ? set_errno(error) : 0);
1444 }
1445
1446 /*ARGSUSED3*/
1447 int
1448 getsockname(int sock, struct sockaddr *name,
1449 socklen_t *namelenp, int version)
1450 {
1451 struct sonode *so;
1452 int error;
1453 socklen_t namelen, sock_addrlen;
1454 struct sockaddr *sock_addrp;
1455
1456 dprint(1, ("getsockname(%d, %p, %p)\n",
1457 sock, (void *)name, (void *)namelenp));
1458
1459 if ((so = getsonode(sock, &error, NULL)) == NULL)
1460 goto bad;
1461
1462 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1463 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1464 (name == NULL && namelen != 0)) {
1465 error = EFAULT;
1466 goto rel_out;
1467 }
1468
1469 sock_addrlen = so->so_max_addr_len;
1470 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1471 if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen,
1472 CRED())) == 0) {
1473 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1474 ASSERT(sock_addrlen <= so->so_max_addr_len);
1475 error = copyout_name(name, namelen, namelenp,
1476 (void *)sock_addrp, sock_addrlen);
1477 }
1478 kmem_free(sock_addrp, so->so_max_addr_len);
1479 rel_out:
1480 releasef(sock);
1481 bad: return (error != 0 ? set_errno(error) : 0);
1482 }
1483
1484 /*ARGSUSED5*/
1485 int
1486 getsockopt(int sock,
1487 int level,
1488 int option_name,
1489 void *option_value,
1490 socklen_t *option_lenp,
1491 int version)
1492 {
1493 struct sonode *so;
1494 socklen_t optlen, optlen_res;
1495 void *optval;
1496 int error;
1497
1498 dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n",
1499 sock, level, option_name, option_value, (void *)option_lenp));
1500
1501 if ((so = getsonode(sock, &error, NULL)) == NULL)
1502 return (set_errno(error));
1503
1504 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1505 if (copyin(option_lenp, &optlen, sizeof (optlen))) {
1506 releasef(sock);
1507 return (set_errno(EFAULT));
1508 }
1509 /*
1510 * Verify that the length is not excessive to prevent
1511 * an application from consuming all of kernel memory.
1512 */
1513 if (optlen > SO_MAXARGSIZE) {
1514 error = EINVAL;
1515 releasef(sock);
1516 return (set_errno(error));
1517 }
1518 optval = kmem_alloc(optlen, KM_SLEEP);
1519 optlen_res = optlen;
1520 error = socket_getsockopt(so, level, option_name, optval,
1521 &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2,
1522 CRED());
1523 releasef(sock);
1524 if (error) {
1525 kmem_free(optval, optlen);
1526 return (set_errno(error));
1527 }
1528 error = copyout_arg(option_value, optlen, option_lenp,
1529 optval, optlen_res);
1530 kmem_free(optval, optlen);
1531 if (error)
1532 return (set_errno(error));
1533 return (0);
1534 }
1535
1536 /*ARGSUSED5*/
1537 int
1538 setsockopt(int sock,
1539 int level,
1540 int option_name,
1541 void *option_value,
1542 socklen_t option_len,
1543 int version)
1544 {
1545 struct sonode *so;
1546 intptr_t buffer[2];
1547 void *optval = NULL;
1548 int error;
1549
1550 dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n",
1551 sock, level, option_name, option_value, option_len));
1552
1553 if ((so = getsonode(sock, &error, NULL)) == NULL)
1554 return (set_errno(error));
1555
1556 if (option_value != NULL) {
1557 if (option_len != 0) {
1558 /*
1559 * Verify that the length is not excessive to prevent
1560 * an application from consuming all of kernel memory.
1561 */
1562 if (option_len > SO_MAXARGSIZE) {
1563 error = EINVAL;
1564 goto done2;
1565 }
1566 optval = option_len <= sizeof (buffer) ?
1567 &buffer : kmem_alloc((size_t)option_len, KM_SLEEP);
1568 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1569 if (copyin(option_value, optval, (size_t)option_len)) {
1570 error = EFAULT;
1571 goto done1;
1572 }
1573 }
1574 } else
1575 option_len = 0;
1576
1577 error = socket_setsockopt(so, level, option_name, optval,
1578 (t_uscalar_t)option_len, CRED());
1579 done1:
1580 if (optval != buffer)
1581 kmem_free(optval, (size_t)option_len);
1582 done2:
1583 releasef(sock);
1584 if (error)
1585 return (set_errno(error));
1586 return (0);
1587 }
1588
1589 static int
1590 sockconf_add_sock(int family, int type, int protocol, char *name)
1591 {
1592 int error = 0;
1593 char *kdevpath = NULL;
1594 char *kmodule = NULL;
1595 char *buf = NULL;
1596 size_t pathlen = 0;
1597 struct sockparams *sp;
1598
1599 if (name == NULL)
1600 return (EINVAL);
1601 /*
1602 * Copyin the name.
1603 * This also makes it possible to check for too long pathnames.
1604 * Compress the space needed for the name before passing it
1605 * to soconfig - soconfig will store the string until
1606 * the configuration is removed.
1607 */
1608 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1609 if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) {
1610 kmem_free(buf, MAXPATHLEN);
1611 return (error);
1612 }
1613 if (strncmp(buf, "/dev", strlen("/dev")) == 0) {
1614 /* For device */
1615
1616 /*
1617 * Special handling for NCA:
1618 *
1619 * DEV_NCA is never opened even if an application
1620 * requests for AF_NCA. The device opened is instead a
1621 * predefined AF_INET transport (NCA_INET_DEV).
1622 *
1623 * Prior to Volo (PSARC/2007/587) NCA would determine
1624 * the device using a lookup, which worked then because
1625 * all protocols were based on TPI. Since TPI is no
1626 * longer the default, we have to explicitly state
1627 * which device to use.
1628 */
1629 if (strcmp(buf, NCA_DEV) == 0) {
1630 /* only support entry <28, 2, 0> */
1631 if (family != AF_NCA || type != SOCK_STREAM ||
1632 protocol != 0) {
1633 kmem_free(buf, MAXPATHLEN);
1634 return (EINVAL);
1635 }
1636
1637 pathlen = strlen(NCA_INET_DEV) + 1;
1638 kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1639 bcopy(NCA_INET_DEV, kdevpath, pathlen);
1640 kdevpath[pathlen - 1] = '\0';
1641 } else {
1642 kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1643 bcopy(buf, kdevpath, pathlen);
1644 kdevpath[pathlen - 1] = '\0';
1645 }
1646 } else {
1647 /* For socket module */
1648 kmodule = kmem_alloc(pathlen, KM_SLEEP);
1649 bcopy(buf, kmodule, pathlen);
1650 kmodule[pathlen - 1] = '\0';
1651 pathlen = 0;
1652 }
1653 kmem_free(buf, MAXPATHLEN);
1654
1655 /* sockparams_create frees mod name and devpath upon failure */
1656 sp = sockparams_create(family, type, protocol, kmodule,
1657 kdevpath, pathlen, 0, KM_SLEEP, &error);
1658 if (sp != NULL) {
1659 error = sockparams_add(sp);
1660 if (error != 0)
1661 sockparams_destroy(sp);
1662 }
1663
1664 return (error);
1665 }
1666
1667 static int
1668 sockconf_remove_sock(int family, int type, int protocol)
1669 {
1670 return (sockparams_delete(family, type, protocol));
1671 }
1672
1673 static int
1674 sockconfig_remove_filter(const char *uname)
1675 {
1676 char kname[SOF_MAXNAMELEN];
1677 size_t len;
1678 int error;
1679 sof_entry_t *ent;
1680
1681 if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0)
1682 return (error);
1683
1684 ent = sof_entry_remove_by_name(kname);
1685 if (ent == NULL)
1686 return (ENXIO);
1687
1688 mutex_enter(&ent->sofe_lock);
1689 ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED));
1690 if (ent->sofe_refcnt == 0) {
1691 mutex_exit(&ent->sofe_lock);
1692 sof_entry_free(ent);
1693 } else {
1694 /* let the last socket free the filter */
1695 ent->sofe_flags |= SOFEF_CONDEMED;
1696 mutex_exit(&ent->sofe_lock);
1697 }
1698
1699 return (0);
1700 }
1701
1702 static int
1703 sockconfig_add_filter(const char *uname, void *ufilpropp)
1704 {
1705 struct sockconfig_filter_props filprop;
1706 sof_entry_t *ent;
1707 int error;
1708 size_t tuplesz, len;
1709 char hintbuf[SOF_MAXNAMELEN];
1710
1711 ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP);
1712 mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL);
1713
1714 if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN,
1715 &len)) != 0) {
1716 sof_entry_free(ent);
1717 return (error);
1718 }
1719
1720 if (get_udatamodel() == DATAMODEL_NATIVE) {
1721 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) {
1722 sof_entry_free(ent);
1723 return (EFAULT);
1724 }
1725 }
1726 #ifdef _SYSCALL32_IMPL
1727 else {
1728 struct sockconfig_filter_props32 filprop32;
1729
1730 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) {
1731 sof_entry_free(ent);
1732 return (EFAULT);
1733 }
1734 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname;
1735 filprop.sfp_autoattach = filprop32.sfp_autoattach;
1736 filprop.sfp_hint = filprop32.sfp_hint;
1737 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg;
1738 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt;
1739 filprop.sfp_socktuple =
1740 (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple;
1741 }
1742 #endif /* _SYSCALL32_IMPL */
1743
1744 if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname,
1745 sizeof (ent->sofe_modname), &len)) != 0) {
1746 sof_entry_free(ent);
1747 return (error);
1748 }
1749
1750 /*
1751 * A filter must specify at least one socket tuple.
1752 */
1753 if (filprop.sfp_socktuple_cnt == 0 ||
1754 filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) {
1755 sof_entry_free(ent);
1756 return (EINVAL);
1757 }
1758 ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG;
1759 ent->sofe_hint = filprop.sfp_hint;
1760
1761 /*
1762 * Verify the hint, and copy in the hint argument, if necessary.
1763 */
1764 switch (ent->sofe_hint) {
1765 case SOF_HINT_BEFORE:
1766 case SOF_HINT_AFTER:
1767 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf,
1768 sizeof (hintbuf), &len)) != 0) {
1769 sof_entry_free(ent);
1770 return (error);
1771 }
1772 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP);
1773 bcopy(hintbuf, ent->sofe_hintarg, len);
1774 /* FALLTHRU */
1775 case SOF_HINT_TOP:
1776 case SOF_HINT_BOTTOM:
1777 /* hints cannot be used with programmatic filters */
1778 if (ent->sofe_flags & SOFEF_PROG) {
1779 sof_entry_free(ent);
1780 return (EINVAL);
1781 }
1782 break;
1783 case SOF_HINT_NONE:
1784 break;
1785 default:
1786 /* bad hint value */
1787 sof_entry_free(ent);
1788 return (EINVAL);
1789 }
1790
1791 ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt;
1792 tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt;
1793 ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP);
1794
1795 if (get_udatamodel() == DATAMODEL_NATIVE) {
1796 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple,
1797 tuplesz)) {
1798 sof_entry_free(ent);
1799 return (EFAULT);
1800 }
1801 }
1802 #ifdef _SYSCALL32_IMPL
1803 else {
1804 int i;
1805 caddr_t data = (caddr_t)filprop.sfp_socktuple;
1806 sof_socktuple_t *tup = ent->sofe_socktuple;
1807 sof_socktuple32_t tup32;
1808
1809 tup = ent->sofe_socktuple;
1810 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) {
1811 ASSERT(tup < ent->sofe_socktuple + tuplesz);
1812
1813 if (copyin(data, &tup32, sizeof (tup32)) != 0) {
1814 sof_entry_free(ent);
1815 return (EFAULT);
1816 }
1817 tup->sofst_family = tup32.sofst_family;
1818 tup->sofst_type = tup32.sofst_type;
1819 tup->sofst_protocol = tup32.sofst_protocol;
1820
1821 data += sizeof (tup32);
1822 }
1823 }
1824 #endif /* _SYSCALL32_IMPL */
1825
1826 /* Sockets can start using the filter as soon as the filter is added */
1827 if ((error = sof_entry_add(ent)) != 0)
1828 sof_entry_free(ent);
1829
1830 return (error);
1831 }
1832
1833 /*
1834 * Socket configuration system call. It is used to add and remove
1835 * socket types.
1836 */
1837 int
1838 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4)
1839 {
1840 int error = 0;
1841
1842 if (secpolicy_net_config(CRED(), B_FALSE) != 0)
1843 return (set_errno(EPERM));
1844
1845 if (sockfs_defer_nl7c_init) {
1846 nl7c_init();
1847 sockfs_defer_nl7c_init = 0;
1848 }
1849
1850 switch (cmd) {
1851 case SOCKCONFIG_ADD_SOCK:
1852 error = sockconf_add_sock((int)(uintptr_t)arg1,
1853 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4);
1854 break;
1855 case SOCKCONFIG_REMOVE_SOCK:
1856 error = sockconf_remove_sock((int)(uintptr_t)arg1,
1857 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3);
1858 break;
1859 case SOCKCONFIG_ADD_FILTER:
1860 error = sockconfig_add_filter((const char *)arg1, arg2);
1861 break;
1862 case SOCKCONFIG_REMOVE_FILTER:
1863 error = sockconfig_remove_filter((const char *)arg1);
1864 break;
1865 default:
1866 #ifdef DEBUG
1867 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd);
1868 #endif
1869 error = EINVAL;
1870 break;
1871 }
1872
1873 if (error != 0) {
1874 eprintline(error);
1875 return (set_errno(error));
1876 }
1877 return (0);
1878 }
1879
1880
1881 /*
1882 * Sendfile is implemented through two schemes, direct I/O or by
1883 * caching in the filesystem page cache. We cache the input file by
1884 * default and use direct I/O only if sendfile_max_size is set
1885 * appropriately as explained below. Note that this logic is consistent
1886 * with other filesystems where caching is turned on by default
1887 * unless explicitly turned off by using the DIRECTIO ioctl.
1888 *
1889 * We choose a slightly different scheme here. One can turn off
1890 * caching by setting sendfile_max_size to 0. One can also enable
1891 * caching of files <= sendfile_max_size by setting sendfile_max_size
1892 * to an appropriate value. By default sendfile_max_size is set to the
1893 * maximum value so that all files are cached. In future, we may provide
1894 * better interfaces for caching the file.
1895 *
1896 * Sendfile through Direct I/O (Zero copy)
1897 * --------------------------------------
1898 *
1899 * As disks are normally slower than the network, we can't have a
1900 * single thread that reads the disk and writes to the network. We
1901 * need to have parallelism. This is done by having the sendfile
1902 * thread create another thread that reads from the filesystem
1903 * and queues it for network processing. In this scheme, the data
1904 * is never copied anywhere i.e it is zero copy unlike the other
1905 * scheme.
1906 *
1907 * We have a sendfile queue (snfq) where each sendfile
1908 * request (snf_req_t) is queued for processing by a thread. Number
1909 * of threads is dynamically allocated and they exit if they are idling
1910 * beyond a specified amount of time. When each request (snf_req_t) is
1911 * processed by a thread, it produces a number of mblk_t structures to
1912 * be consumed by the sendfile thread. snf_deque and snf_enque are
1913 * used for consuming and producing mblks. Size of the filesystem
1914 * read is determined by the tunable (sendfile_read_size). A single
1915 * mblk holds sendfile_read_size worth of data (except the last
1916 * read of the file) which is sent down as a whole to the network.
1917 * sendfile_read_size is set to 1 MB as this seems to be the optimal
1918 * value for the UFS filesystem backed by a striped storage array.
1919 *
1920 * Synchronisation between read (producer) and write (consumer) threads.
1921 * --------------------------------------------------------------------
1922 *
1923 * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while
1924 * adding and deleting items in this list. Error can happen anytime
1925 * during read or write. There could be unprocessed mblks in the
1926 * sr_ib_XXX list when a read or write error occurs. Whenever error
1927 * is encountered, we need two things to happen :
1928 *
1929 * a) One of the threads need to clean the mblks.
1930 * b) When one thread encounters an error, the other should stop.
1931 *
1932 * For (a), we don't want to penalize the reader thread as it could do
1933 * some useful work processing other requests. For (b), the error can
1934 * be detected by examining sr_read_error or sr_write_error.
1935 * sr_lock protects sr_read_error and sr_write_error. If both reader and
1936 * writer encounters error, we need to report the write error back to
1937 * the application as that's what would have happened if the operations
1938 * were done sequentially. With this in mind, following should work :
1939 *
1940 * - Check for errors before read or write.
1941 * - If the reader encounters error, set the error in sr_read_error.
1942 * Check sr_write_error, if it is set, send cv_signal as it is
1943 * waiting for reader to complete. If it is not set, the writer
1944 * is either running sinking data to the network or blocked
1945 * because of flow control. For handling the latter case, we
1946 * always send a signal. In any case, it will examine sr_read_error
1947 * and return. sr_read_error is marked with SR_READ_DONE to tell
1948 * the writer that the reader is done in all the cases.
1949 * - If the writer encounters error, set the error in sr_write_error.
1950 * The reader thread is either blocked because of flow control or
1951 * running reading data from the disk. For the former, we need to
1952 * wakeup the thread. Again to keep it simple, we always wake up
1953 * the reader thread. Then, wait for the read thread to complete
1954 * if it is not done yet. Cleanup and return.
1955 *
1956 * High and low water marks for the read thread.
1957 * --------------------------------------------
1958 *
1959 * If sendfile() is used to send data over a slow network, we need to
1960 * make sure that the read thread does not produce data at a faster
1961 * rate than the network. This can happen if the disk is faster than
1962 * the network. In such a case, we don't want to build a very large queue.
1963 * But we would still like to get all of the network throughput possible.
1964 * This implies that network should never block waiting for data.
1965 * As there are lot of disk throughput/network throughput combinations
1966 * possible, it is difficult to come up with an accurate number.
1967 * A typical 10K RPM disk has a max seek latency 17ms and rotational
1968 * latency of 3ms for reading a disk block. Thus, the total latency to
1969 * initiate a new read, transfer data from the disk and queue for
1970 * transmission would take about a max of 25ms. Todays max transfer rate
1971 * for network is 100MB/sec. If the thread is blocked because of flow
1972 * control, it would take 25ms to get new data ready for transmission.
1973 * We have to make sure that network is not idling, while we are initiating
1974 * new transfers. So, at 100MB/sec, to keep network busy we would need
1975 * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data.
1976 * We need to pick a high water mark so that the woken up thread would
1977 * do considerable work before blocking again to prevent thrashing. Currently,
1978 * we pick this to be 10 times that of the low water mark.
1979 *
1980 * Sendfile with segmap caching (One copy from page cache to mblks).
1981 * ----------------------------------------------------------------
1982 *
1983 * We use the segmap cache for caching the file, if the size of file
1984 * is <= sendfile_max_size. In this case we don't use threads as VM
1985 * is reasonably fast enough to keep up with the network. If the underlying
1986 * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth
1987 * of data into segmap space, and use the virtual address from segmap
1988 * directly through desballoc() to avoid copy. Once the transport is done
1989 * with the data, the mapping will be released through segmap_release()
1990 * called by the call-back routine.
1991 *
1992 * If zero-copy is not allowed by the transport, we simply call VOP_READ()
1993 * to copy the data from the filesystem into our temporary network buffer.
1994 *
1995 * To disable caching, set sendfile_max_size to 0.
1996 */
1997
1998 uint_t sendfile_read_size = 1024 * 1024;
1999 #define SENDFILE_REQ_LOWAT 3 * 1024 * 1024
2000 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT;
2001 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT;
2002 struct sendfile_stats sf_stats;
2003 struct sendfile_queue *snfq;
2004 clock_t snfq_timeout;
2005 off64_t sendfile_max_size;
2006
2007 static void snf_enque(snf_req_t *, mblk_t *);
2008 static mblk_t *snf_deque(snf_req_t *);
2009
2010 void
2011 sendfile_init(void)
2012 {
2013 snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP);
2014
2015 mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL);
2016 cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL);
2017 snfq->snfq_max_threads = max_ncpus;
2018 snfq_timeout = SNFQ_TIMEOUT;
2019 /* Cache all files by default. */
2020 sendfile_max_size = MAXOFFSET_T;
2021 }
2022
2023 /*
2024 * Queues a mblk_t for network processing.
2025 */
2026 static void
2027 snf_enque(snf_req_t *sr, mblk_t *mp)
2028 {
2029 mp->b_next = NULL;
2030 mutex_enter(&sr->sr_lock);
2031 if (sr->sr_mp_head == NULL) {
2032 sr->sr_mp_head = sr->sr_mp_tail = mp;
2033 cv_signal(&sr->sr_cv);
2034 } else {
2035 sr->sr_mp_tail->b_next = mp;
2036 sr->sr_mp_tail = mp;
2037 }
2038 sr->sr_qlen += MBLKL(mp);
2039 while ((sr->sr_qlen > sr->sr_hiwat) &&
2040 (sr->sr_write_error == 0)) {
2041 sf_stats.ss_full_waits++;
2042 cv_wait(&sr->sr_cv, &sr->sr_lock);
2043 }
2044 mutex_exit(&sr->sr_lock);
2045 }
2046
2047 /*
2048 * De-queues a mblk_t for network processing.
2049 */
2050 static mblk_t *
2051 snf_deque(snf_req_t *sr)
2052 {
2053 mblk_t *mp;
2054
2055 mutex_enter(&sr->sr_lock);
2056 /*
2057 * If we have encountered an error on read or read is
2058 * completed and no more mblks, return NULL.
2059 * We need to check for NULL sr_mp_head also as
2060 * the reads could have completed and there is
2061 * nothing more to come.
2062 */
2063 if (((sr->sr_read_error & ~SR_READ_DONE) != 0) ||
2064 ((sr->sr_read_error & SR_READ_DONE) &&
2065 sr->sr_mp_head == NULL)) {
2066 mutex_exit(&sr->sr_lock);
2067 return (NULL);
2068 }
2069 /*
2070 * To start with neither SR_READ_DONE is marked nor
2071 * the error is set. When we wake up from cv_wait,
2072 * following are the possibilities :
2073 *
2074 * a) sr_read_error is zero and mblks are queued.
2075 * b) sr_read_error is set to SR_READ_DONE
2076 * and mblks are queued.
2077 * c) sr_read_error is set to SR_READ_DONE
2078 * and no mblks.
2079 * d) sr_read_error is set to some error other
2080 * than SR_READ_DONE.
2081 */
2082
2083 while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) {
2084 sf_stats.ss_empty_waits++;
2085 cv_wait(&sr->sr_cv, &sr->sr_lock);
2086 }
2087 /* Handle (a) and (b) first - the normal case. */
2088 if (((sr->sr_read_error & ~SR_READ_DONE) == 0) &&
2089 (sr->sr_mp_head != NULL)) {
2090 mp = sr->sr_mp_head;
2091 sr->sr_mp_head = mp->b_next;
2092 sr->sr_qlen -= MBLKL(mp);
2093 if (sr->sr_qlen < sr->sr_lowat)
2094 cv_signal(&sr->sr_cv);
2095 mutex_exit(&sr->sr_lock);
2096 mp->b_next = NULL;
2097 return (mp);
2098 }
2099 /* Handle (c) and (d). */
2100 mutex_exit(&sr->sr_lock);
2101 return (NULL);
2102 }
2103
2104 /*
2105 * Reads data from the filesystem and queues it for network processing.
2106 */
2107 void
2108 snf_async_read(snf_req_t *sr)
2109 {
2110 size_t iosize;
2111 u_offset_t fileoff;
2112 u_offset_t size;
2113 int ret_size;
2114 int error;
2115 file_t *fp;
2116 mblk_t *mp;
2117 struct vnode *vp;
2118 int extra = 0;
2119 int maxblk = 0;
2120 int wroff = 0;
2121 struct sonode *so;
2122
2123 fp = sr->sr_fp;
2124 size = sr->sr_file_size;
2125 fileoff = sr->sr_file_off;
2126
2127 /*
2128 * Ignore the error for filesystems that doesn't support DIRECTIO.
2129 */
2130 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0,
2131 kcred, NULL, NULL);
2132
2133 vp = sr->sr_vp;
2134 if (vp->v_type == VSOCK) {
2135 stdata_t *stp;
2136
2137 /*
2138 * Get the extra space to insert a header and a trailer.
2139 */
2140 so = VTOSO(vp);
2141 stp = vp->v_stream;
2142 if (stp == NULL) {
2143 wroff = so->so_proto_props.sopp_wroff;
2144 maxblk = so->so_proto_props.sopp_maxblk;
2145 extra = wroff + so->so_proto_props.sopp_tail;
2146 } else {
2147 wroff = (int)(stp->sd_wroff);
2148 maxblk = (int)(stp->sd_maxblk);
2149 extra = wroff + (int)(stp->sd_tail);
2150 }
2151 }
2152
2153 while ((size != 0) && (sr->sr_write_error == 0)) {
2154
2155 iosize = (int)MIN(sr->sr_maxpsz, size);
2156
2157 /*
2158 * Socket filters can limit the mblk size,
2159 * so limit reads to maxblk if there are
2160 * filters present.
2161 */
2162 if (vp->v_type == VSOCK &&
2163 so->so_filter_active > 0 && maxblk != INFPSZ)
2164 iosize = (int)MIN(iosize, maxblk);
2165
2166 if (is_system_labeled()) {
2167 mp = allocb_cred(iosize + extra, CRED(),
2168 curproc->p_pid);
2169 } else {
2170 mp = allocb(iosize + extra, BPRI_MED);
2171 }
2172 if (mp == NULL) {
2173 error = EAGAIN;
2174 break;
2175 }
2176
2177 mp->b_rptr += wroff;
2178
2179 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize);
2180
2181 /* Error or Reached EOF ? */
2182 if ((error != 0) || (ret_size == 0)) {
2183 freeb(mp);
2184 break;
2185 }
2186 mp->b_wptr = mp->b_rptr + ret_size;
2187
2188 snf_enque(sr, mp);
2189 size -= ret_size;
2190 fileoff += ret_size;
2191 }
2192 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0,
2193 kcred, NULL, NULL);
2194 mutex_enter(&sr->sr_lock);
2195 sr->sr_read_error = error;
2196 sr->sr_read_error |= SR_READ_DONE;
2197 cv_signal(&sr->sr_cv);
2198 mutex_exit(&sr->sr_lock);
2199 }
2200
2201 void
2202 snf_async_thread(void)
2203 {
2204 snf_req_t *sr;
2205 callb_cpr_t cprinfo;
2206 clock_t time_left = 1;
2207
2208 CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq");
2209
2210 mutex_enter(&snfq->snfq_lock);
2211 for (;;) {
2212 /*
2213 * If we didn't find a entry, then block until woken up
2214 * again and then look through the queues again.
2215 */
2216 while ((sr = snfq->snfq_req_head) == NULL) {
2217 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2218 if (time_left <= 0) {
2219 snfq->snfq_svc_threads--;
2220 CALLB_CPR_EXIT(&cprinfo);
2221 thread_exit();
2222 /* NOTREACHED */
2223 }
2224 snfq->snfq_idle_cnt++;
2225
2226 time_left = cv_reltimedwait(&snfq->snfq_cv,
2227 &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK);
2228 snfq->snfq_idle_cnt--;
2229
2230 CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock);
2231 }
2232 snfq->snfq_req_head = sr->sr_next;
2233 snfq->snfq_req_cnt--;
2234 mutex_exit(&snfq->snfq_lock);
2235 snf_async_read(sr);
2236 mutex_enter(&snfq->snfq_lock);
2237 }
2238 }
2239
2240
2241 snf_req_t *
2242 create_thread(int operation, struct vnode *vp, file_t *fp,
2243 u_offset_t fileoff, u_offset_t size)
2244 {
2245 snf_req_t *sr;
2246 stdata_t *stp;
2247
2248 sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP);
2249
2250 sr->sr_vp = vp;
2251 sr->sr_fp = fp;
2252 stp = vp->v_stream;
2253
2254 /*
2255 * store sd_qn_maxpsz into sr_maxpsz while we have stream head.
2256 * stream might be closed before thread returns from snf_async_read.
2257 */
2258 if (stp != NULL && stp->sd_qn_maxpsz > 0) {
2259 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz);
2260 } else {
2261 sr->sr_maxpsz = MAXBSIZE;
2262 }
2263
2264 sr->sr_operation = operation;
2265 sr->sr_file_off = fileoff;
2266 sr->sr_file_size = size;
2267 sr->sr_hiwat = sendfile_req_hiwat;
2268 sr->sr_lowat = sendfile_req_lowat;
2269 mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL);
2270 cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL);
2271 /*
2272 * See whether we need another thread for servicing this
2273 * request. If there are already enough requests queued
2274 * for the threads, create one if not exceeding
2275 * snfq_max_threads.
2276 */
2277 mutex_enter(&snfq->snfq_lock);
2278 if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt &&
2279 snfq->snfq_svc_threads < snfq->snfq_max_threads) {
2280 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0,
2281 TS_RUN, minclsyspri);
2282 snfq->snfq_svc_threads++;
2283 }
2284 if (snfq->snfq_req_head == NULL) {
2285 snfq->snfq_req_head = snfq->snfq_req_tail = sr;
2286 cv_signal(&snfq->snfq_cv);
2287 } else {
2288 snfq->snfq_req_tail->sr_next = sr;
2289 snfq->snfq_req_tail = sr;
2290 }
2291 snfq->snfq_req_cnt++;
2292 mutex_exit(&snfq->snfq_lock);
2293 return (sr);
2294 }
2295
2296 int
2297 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size,
2298 ssize_t *count)
2299 {
2300 snf_req_t *sr;
2301 mblk_t *mp;
2302 int iosize;
2303 int error = 0;
2304 short fflag;
2305 struct vnode *vp;
2306 int ksize;
2307 struct nmsghdr msg;
2308
2309 ksize = 0;
2310 *count = 0;
2311 bzero(&msg, sizeof (msg));
2312
2313 vp = fp->f_vnode;
2314 fflag = fp->f_flag;
2315 if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL)
2316 return (EAGAIN);
2317
2318 /*
2319 * We check for read error in snf_deque. It has to check
2320 * for successful READ_DONE and return NULL, and we might
2321 * as well make an additional check there.
2322 */
2323 while ((mp = snf_deque(sr)) != NULL) {
2324
2325 if (ISSIG(curthread, JUSTLOOKING)) {
2326 freeb(mp);
2327 error = EINTR;
2328 break;
2329 }
2330 iosize = MBLKL(mp);
2331
2332 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2333
2334 if (error != 0) {
2335 if (mp != NULL)
2336 freeb(mp);
2337 break;
2338 }
2339 ksize += iosize;
2340 }
2341 *count = ksize;
2342
2343 mutex_enter(&sr->sr_lock);
2344 sr->sr_write_error = error;
2345 /* Look at the big comments on why we cv_signal here. */
2346 cv_signal(&sr->sr_cv);
2347
2348 /* Wait for the reader to complete always. */
2349 while (!(sr->sr_read_error & SR_READ_DONE)) {
2350 cv_wait(&sr->sr_cv, &sr->sr_lock);
2351 }
2352 /* If there is no write error, check for read error. */
2353 if (error == 0)
2354 error = (sr->sr_read_error & ~SR_READ_DONE);
2355
2356 if (error != 0) {
2357 mblk_t *next_mp;
2358
2359 mp = sr->sr_mp_head;
2360 while (mp != NULL) {
2361 next_mp = mp->b_next;
2362 mp->b_next = NULL;
2363 freeb(mp);
2364 mp = next_mp;
2365 }
2366 }
2367 mutex_exit(&sr->sr_lock);
2368 kmem_free(sr, sizeof (snf_req_t));
2369 return (error);
2370 }
2371
2372 /* Maximum no.of pages allocated by vpm for sendfile at a time */
2373 #define SNF_VPMMAXPGS (VPMMAXPGS/2)
2374
2375 /*
2376 * Maximum no.of elements in the list returned by vpm, including
2377 * NULL for the last entry
2378 */
2379 #define SNF_MAXVMAPS (SNF_VPMMAXPGS + 1)
2380
2381 typedef struct {
2382 unsigned int snfv_ref;
2383 frtn_t snfv_frtn;
2384 vnode_t *snfv_vp;
2385 struct vmap snfv_vml[SNF_MAXVMAPS];
2386 } snf_vmap_desbinfo;
2387
2388 typedef struct {
2389 frtn_t snfi_frtn;
2390 caddr_t snfi_base;
2391 uint_t snfi_mapoff;
2392 size_t snfi_len;
2393 vnode_t *snfi_vp;
2394 } snf_smap_desbinfo;
2395
2396 /*
2397 * The callback function used for vpm mapped mblks called when the last ref of
2398 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2399 * can be the driver too due to lazy reclaim.
2400 */
2401 void
2402 snf_vmap_desbfree(snf_vmap_desbinfo *snfv)
2403 {
2404 ASSERT(snfv->snfv_ref != 0);
2405 if (atomic_add_32_nv(&snfv->snfv_ref, -1) == 0) {
2406 vpm_unmap_pages(snfv->snfv_vml, S_READ);
2407 VN_RELE(snfv->snfv_vp);
2408 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2409 }
2410 }
2411
2412 /*
2413 * The callback function used for segmap'ped mblks called when the last ref of
2414 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2415 * can be the driver too due to lazy reclaim.
2416 */
2417 void
2418 snf_smap_desbfree(snf_smap_desbinfo *snfi)
2419 {
2420 if (! IS_KPM_ADDR(snfi->snfi_base)) {
2421 /*
2422 * We don't need to call segmap_fault(F_SOFTUNLOCK) for
2423 * segmap_kpm as long as the latter never falls back to
2424 * "use_segmap_range". (See segmap_getmapflt().)
2425 *
2426 * Using S_OTHER saves an redundant hat_setref() in
2427 * segmap_unlock()
2428 */
2429 (void) segmap_fault(kas.a_hat, segkmap,
2430 (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base +
2431 snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len,
2432 F_SOFTUNLOCK, S_OTHER);
2433 }
2434 (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED);
2435 VN_RELE(snfi->snfi_vp);
2436 kmem_free(snfi, sizeof (*snfi));
2437 }
2438
2439 /*
2440 * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk.
2441 * When segmap is used, the mblk contains a segmap slot of no more
2442 * than MAXBSIZE.
2443 *
2444 * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained
2445 * in each iteration and sent by socket_sendmblk until an error occurs or
2446 * the requested size has been transferred. An mblk is esballoca'ed from
2447 * each mapped page and a chain of these mblk is sent to the transport layer.
2448 * vpm will be called to unmap the pages when all mblks have been freed by
2449 * free_func.
2450 *
2451 * At the end of the whole sendfile() operation, we wait till the data from
2452 * the last mblk is ack'ed by the transport before returning so that the
2453 * caller of sendfile() can safely modify the file content.
2454 */
2455 int
2456 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size,
2457 ssize_t *count, boolean_t nowait)
2458 {
2459 caddr_t base;
2460 int mapoff;
2461 vnode_t *vp;
2462 mblk_t *mp = NULL;
2463 int chain_size;
2464 int error;
2465 clock_t deadlk_wait;
2466 short fflag;
2467 int ksize;
2468 struct vattr va;
2469 boolean_t dowait = B_FALSE;
2470 struct nmsghdr msg;
2471
2472 vp = fp->f_vnode;
2473 fflag = fp->f_flag;
2474 ksize = 0;
2475 bzero(&msg, sizeof (msg));
2476
2477 for (;;) {
2478 if (ISSIG(curthread, JUSTLOOKING)) {
2479 error = EINTR;
2480 break;
2481 }
2482
2483 if (vpm_enable) {
2484 snf_vmap_desbinfo *snfv;
2485 mblk_t *nmp;
2486 int mblk_size;
2487 int maxsize;
2488 int i;
2489
2490 mapoff = fileoff & PAGEOFFSET;
2491 maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size);
2492
2493 snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo),
2494 KM_SLEEP);
2495
2496 /*
2497 * Get vpm mappings for maxsize with read access.
2498 * If the pages aren't available yet, we get
2499 * DEADLK, so wait and try again a little later using
2500 * an increasing wait. We might be here a long time.
2501 *
2502 * If delay_sig returns EINTR, be sure to exit and
2503 * pass it up to the caller.
2504 */
2505 deadlk_wait = 0;
2506 while ((error = vpm_map_pages(fvp, fileoff,
2507 (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml,
2508 SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) {
2509 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2510 if ((error = delay_sig(deadlk_wait)) != 0) {
2511 break;
2512 }
2513 }
2514 if (error != 0) {
2515 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2516 error = (error == EINTR) ? EINTR : EIO;
2517 goto out;
2518 }
2519 snfv->snfv_frtn.free_func = snf_vmap_desbfree;
2520 snfv->snfv_frtn.free_arg = (caddr_t)snfv;
2521
2522 /* Construct the mblk chain from the page mappings */
2523 chain_size = 0;
2524 for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) &&
2525 total_size > 0; i++) {
2526 ASSERT(chain_size < maxsize);
2527 mblk_size = MIN(snfv->snfv_vml[i].vs_len -
2528 mapoff, total_size);
2529 nmp = esballoca(
2530 (uchar_t *)snfv->snfv_vml[i].vs_addr +
2531 mapoff, mblk_size, BPRI_HI,
2532 &snfv->snfv_frtn);
2533
2534 /*
2535 * We return EAGAIN after unmapping the pages
2536 * if we cannot allocate the the head of the
2537 * chain. Otherwise, we continue sending the
2538 * mblks constructed so far.
2539 */
2540 if (nmp == NULL) {
2541 if (i == 0) {
2542 vpm_unmap_pages(snfv->snfv_vml,
2543 S_READ);
2544 kmem_free(snfv,
2545 sizeof (snf_vmap_desbinfo));
2546 error = EAGAIN;
2547 goto out;
2548 }
2549 break;
2550 }
2551 /* Mark this dblk with the zero-copy flag */
2552 nmp->b_datap->db_struioflag |= STRUIO_ZC;
2553 nmp->b_wptr += mblk_size;
2554 chain_size += mblk_size;
2555 fileoff += mblk_size;
2556 total_size -= mblk_size;
2557 snfv->snfv_ref++;
2558 mapoff = 0;
2559 if (i > 0)
2560 linkb(mp, nmp);
2561 else
2562 mp = nmp;
2563 }
2564 VN_HOLD(fvp);
2565 snfv->snfv_vp = fvp;
2566 } else {
2567 /* vpm not supported. fallback to segmap */
2568 snf_smap_desbinfo *snfi;
2569
2570 mapoff = fileoff & MAXBOFFSET;
2571 chain_size = MAXBSIZE - mapoff;
2572 if (chain_size > total_size)
2573 chain_size = total_size;
2574 /*
2575 * we don't forcefault because we'll call
2576 * segmap_fault(F_SOFTLOCK) next.
2577 *
2578 * S_READ will get the ref bit set (by either
2579 * segmap_getmapflt() or segmap_fault()) and page
2580 * shared locked.
2581 */
2582 base = segmap_getmapflt(segkmap, fvp, fileoff,
2583 chain_size, segmap_kpm ? SM_FAULT : 0, S_READ);
2584
2585 snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP);
2586 snfi->snfi_len = (size_t)roundup(mapoff+chain_size,
2587 PAGESIZE)- (mapoff & PAGEMASK);
2588 /*
2589 * We must call segmap_fault() even for segmap_kpm
2590 * because that's how error gets returned.
2591 * (segmap_getmapflt() never fails but segmap_fault()
2592 * does.)
2593 *
2594 * If the pages aren't available yet, we get
2595 * DEADLK, so wait and try again a little later using
2596 * an increasing wait. We might be here a long time.
2597 *
2598 * If delay_sig returns EINTR, be sure to exit and
2599 * pass it up to the caller.
2600 */
2601 deadlk_wait = 0;
2602 while ((error = FC_ERRNO(segmap_fault(kas.a_hat,
2603 segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base +
2604 mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK,
2605 S_READ))) == EDEADLK) {
2606 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2607 if ((error = delay_sig(deadlk_wait)) != 0) {
2608 break;
2609 }
2610 }
2611 if (error != 0) {
2612 (void) segmap_release(segkmap, base, 0);
2613 kmem_free(snfi, sizeof (*snfi));
2614 error = (error == EINTR) ? EINTR : EIO;
2615 goto out;
2616 }
2617 snfi->snfi_frtn.free_func = snf_smap_desbfree;
2618 snfi->snfi_frtn.free_arg = (caddr_t)snfi;
2619 snfi->snfi_base = base;
2620 snfi->snfi_mapoff = mapoff;
2621 mp = esballoca((uchar_t *)base + mapoff, chain_size,
2622 BPRI_HI, &snfi->snfi_frtn);
2623
2624 if (mp == NULL) {
2625 (void) segmap_fault(kas.a_hat, segkmap,
2626 (caddr_t)(uintptr_t)(((uintptr_t)base +
2627 mapoff) & PAGEMASK), snfi->snfi_len,
2628 F_SOFTUNLOCK, S_OTHER);
2629 (void) segmap_release(segkmap, base, 0);
2630 kmem_free(snfi, sizeof (*snfi));
2631 freemsg(mp);
2632 error = EAGAIN;
2633 goto out;
2634 }
2635 VN_HOLD(fvp);
2636 snfi->snfi_vp = fvp;
2637 mp->b_wptr += chain_size;
2638
2639 /* Mark this dblk with the zero-copy flag */
2640 mp->b_datap->db_struioflag |= STRUIO_ZC;
2641 fileoff += chain_size;
2642 total_size -= chain_size;
2643 }
2644
2645 if (total_size == 0 && !nowait) {
2646 ASSERT(!dowait);
2647 dowait = B_TRUE;
2648 mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
2649 }
2650 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2651 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2652 if (error != 0) {
2653 /*
2654 * mp contains the mblks that were not sent by
2655 * socket_sendmblk. Use its size to update *count
2656 */
2657 *count = ksize + (chain_size - msgdsize(mp));
2658 if (mp != NULL)
2659 freemsg(mp);
2660 return (error);
2661 }
2662 ksize += chain_size;
2663 if (total_size == 0)
2664 goto done;
2665
2666 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2667 va.va_mask = AT_SIZE;
2668 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2669 if (error)
2670 break;
2671 /* Read as much as possible. */
2672 if (fileoff >= va.va_size)
2673 break;
2674 if (total_size + fileoff > va.va_size)
2675 total_size = va.va_size - fileoff;
2676 }
2677 out:
2678 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2679 done:
2680 *count = ksize;
2681 if (dowait) {
2682 stdata_t *stp;
2683
2684 stp = vp->v_stream;
2685 if (stp == NULL) {
2686 struct sonode *so;
2687 so = VTOSO(vp);
2688 error = so_zcopy_wait(so);
2689 } else {
2690 mutex_enter(&stp->sd_lock);
2691 while (!(stp->sd_flag & STZCNOTIFY)) {
2692 if (cv_wait_sig(&stp->sd_zcopy_wait,
2693 &stp->sd_lock) == 0) {
2694 error = EINTR;
2695 break;
2696 }
2697 }
2698 stp->sd_flag &= ~STZCNOTIFY;
2699 mutex_exit(&stp->sd_lock);
2700 }
2701 }
2702 return (error);
2703 }
2704
2705 int
2706 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size,
2707 uint_t maxpsz, ssize_t *count)
2708 {
2709 struct vnode *vp;
2710 mblk_t *mp;
2711 int iosize;
2712 int extra = 0;
2713 int error;
2714 short fflag;
2715 int ksize;
2716 int ioflag;
2717 struct uio auio;
2718 struct iovec aiov;
2719 struct vattr va;
2720 int maxblk = 0;
2721 int wroff = 0;
2722 struct sonode *so;
2723 struct nmsghdr msg;
2724
2725 vp = fp->f_vnode;
2726 if (vp->v_type == VSOCK) {
2727 stdata_t *stp;
2728
2729 /*
2730 * Get the extra space to insert a header and a trailer.
2731 */
2732 so = VTOSO(vp);
2733 stp = vp->v_stream;
2734 if (stp == NULL) {
2735 wroff = so->so_proto_props.sopp_wroff;
2736 maxblk = so->so_proto_props.sopp_maxblk;
2737 extra = wroff + so->so_proto_props.sopp_tail;
2738 } else {
2739 wroff = (int)(stp->sd_wroff);
2740 maxblk = (int)(stp->sd_maxblk);
2741 extra = wroff + (int)(stp->sd_tail);
2742 }
2743 }
2744 bzero(&msg, sizeof (msg));
2745 fflag = fp->f_flag;
2746 ksize = 0;
2747 auio.uio_iov = &aiov;
2748 auio.uio_iovcnt = 1;
2749 auio.uio_segflg = UIO_SYSSPACE;
2750 auio.uio_llimit = MAXOFFSET_T;
2751 auio.uio_fmode = fflag;
2752 auio.uio_extflg = UIO_COPY_CACHED;
2753 ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC);
2754 /* If read sync is not asked for, filter sync flags */
2755 if ((ioflag & FRSYNC) == 0)
2756 ioflag &= ~(FSYNC|FDSYNC);
2757 for (;;) {
2758 if (ISSIG(curthread, JUSTLOOKING)) {
2759 error = EINTR;
2760 break;
2761 }
2762 iosize = (int)MIN(maxpsz, size);
2763
2764 /*
2765 * Socket filters can limit the mblk size,
2766 * so limit reads to maxblk if there are
2767 * filters present.
2768 */
2769 if (vp->v_type == VSOCK &&
2770 so->so_filter_active > 0 && maxblk != INFPSZ)
2771 iosize = (int)MIN(iosize, maxblk);
2772
2773 if (is_system_labeled()) {
2774 mp = allocb_cred(iosize + extra, CRED(),
2775 curproc->p_pid);
2776 } else {
2777 mp = allocb(iosize + extra, BPRI_MED);
2778 }
2779 if (mp == NULL) {
2780 error = EAGAIN;
2781 break;
2782 }
2783
2784 mp->b_rptr += wroff;
2785
2786 aiov.iov_base = (caddr_t)mp->b_rptr;
2787 aiov.iov_len = iosize;
2788 auio.uio_loffset = fileoff;
2789 auio.uio_resid = iosize;
2790
2791 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL);
2792 iosize -= auio.uio_resid;
2793
2794 if (error == EINTR && iosize != 0)
2795 error = 0;
2796
2797 if (error != 0 || iosize == 0) {
2798 freeb(mp);
2799 break;
2800 }
2801 mp->b_wptr = mp->b_rptr + iosize;
2802
2803 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2804
2805 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2806
2807 if (error != 0) {
2808 *count = ksize;
2809 if (mp != NULL)
2810 freeb(mp);
2811 return (error);
2812 }
2813 ksize += iosize;
2814 size -= iosize;
2815 if (size == 0)
2816 goto done;
2817
2818 fileoff += iosize;
2819 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2820 va.va_mask = AT_SIZE;
2821 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2822 if (error)
2823 break;
2824 /* Read as much as possible. */
2825 if (fileoff >= va.va_size)
2826 size = 0;
2827 else if (size + fileoff > va.va_size)
2828 size = va.va_size - fileoff;
2829 }
2830 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2831 done:
2832 *count = ksize;
2833 return (error);
2834 }
2835
2836 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
2837 /*
2838 * Largefile support for 32 bit applications only.
2839 */
2840 int
2841 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv,
2842 ssize32_t *count32)
2843 {
2844 ssize32_t sfv_len;
2845 u_offset_t sfv_off, va_size;
2846 struct vnode *vp, *fvp, *realvp;
2847 struct vattr va;
2848 stdata_t *stp;
2849 ssize_t count = 0;
2850 int error = 0;
2851 boolean_t dozcopy = B_FALSE;
2852 uint_t maxpsz;
2853
2854 sfv_len = (ssize32_t)sfv->sfv_len;
2855 if (sfv_len < 0) {
2856 error = EINVAL;
2857 goto out;
2858 }
2859
2860 if (sfv_len == 0) goto out;
2861
2862 sfv_off = (u_offset_t)sfv->sfv_off;
2863
2864 /* Same checks as in pread */
2865 if (sfv_off > MAXOFFSET_T) {
2866 error = EINVAL;
2867 goto out;
2868 }
2869 if (sfv_off + sfv_len > MAXOFFSET_T)
2870 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off);
2871
2872 /*
2873 * There are no more checks on sfv_len. So, we cast it to
2874 * u_offset_t and share the snf_direct_io/snf_cache code between
2875 * 32 bit and 64 bit.
2876 *
2877 * TODO: should do nbl_need_check() like read()?
2878 */
2879 if (sfv_len > sendfile_max_size) {
2880 sf_stats.ss_file_not_cached++;
2881 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len,
2882 &count);
2883 goto out;
2884 }
2885 fvp = rfp->f_vnode;
2886 if (VOP_REALVP(fvp, &realvp, NULL) == 0)
2887 fvp = realvp;
2888 /*
2889 * Grab the lock as a reader to prevent the file size
2890 * from changing underneath.
2891 */
2892 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2893 va.va_mask = AT_SIZE;
2894 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2895 va_size = va.va_size;
2896 if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) {
2897 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2898 goto out;
2899 }
2900 /* Read as much as possible. */
2901 if (sfv_off + sfv_len > va_size)
2902 sfv_len = va_size - sfv_off;
2903
2904 vp = fp->f_vnode;
2905 stp = vp->v_stream;
2906 /*
2907 * When the NOWAIT flag is not set, we enable zero-copy only if the
2908 * transfer size is large enough. This prevents performance loss
2909 * when the caller sends the file piece by piece.
2910 */
2911 if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) ||
2912 (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) &&
2913 !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) {
2914 uint_t copyflag;
2915 copyflag = stp != NULL ? stp->sd_copyflag :
2916 VTOSO(vp)->so_proto_props.sopp_zcopyflag;
2917 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) {
2918 int on = 1;
2919
2920 if (socket_setsockopt(VTOSO(vp), SOL_SOCKET,
2921 SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0)
2922 dozcopy = B_TRUE;
2923 } else {
2924 dozcopy = copyflag & STZCVMSAFE;
2925 }
2926 }
2927 if (dozcopy) {
2928 sf_stats.ss_file_segmap++;
2929 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2930 &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0));
2931 } else {
2932 if (vp->v_type == VSOCK && stp == NULL) {
2933 sonode_t *so = VTOSO(vp);
2934 maxpsz = so->so_proto_props.sopp_maxpsz;
2935 } else if (stp != NULL) {
2936 maxpsz = stp->sd_qn_maxpsz;
2937 } else {
2938 maxpsz = maxphys;
2939 }
2940
2941 if (maxpsz == INFPSZ)
2942 maxpsz = maxphys;
2943 else
2944 maxpsz = roundup(maxpsz, MAXBSIZE);
2945 sf_stats.ss_file_cached++;
2946 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2947 maxpsz, &count);
2948 }
2949 out:
2950 releasef(sfv->sfv_fd);
2951 *count32 = (ssize32_t)count;
2952 return (error);
2953 }
2954 #endif
2955
2956 #ifdef _SYSCALL32_IMPL
2957 /*
2958 * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a
2959 * ssize_t rather than ssize32_t; see the comments above read32 for details.
2960 */
2961
2962 ssize_t
2963 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
2964 {
2965 return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
2966 }
2967
2968 ssize_t
2969 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
2970 caddr32_t name, caddr32_t namelenp)
2971 {
2972 return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
2973 (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp));
2974 }
2975
2976 ssize_t
2977 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
2978 {
2979 return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
2980 }
2981
2982 ssize_t
2983 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
2984 caddr32_t name, socklen_t namelen)
2985 {
2986 return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
2987 (void *)(uintptr_t)name, namelen));
2988 }
2989 #endif /* _SYSCALL32_IMPL */
2990
2991 /*
2992 * Function wrappers (mostly around the sonode switch) for
2993 * backward compatibility.
2994 */
2995
2996 int
2997 soaccept(struct sonode *so, int fflag, struct sonode **nsop)
2998 {
2999 return (socket_accept(so, fflag, CRED(), nsop));
3000 }
3001
3002 int
3003 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3004 int backlog, int flags)
3005 {
3006 int error;
3007
3008 error = socket_bind(so, name, namelen, flags, CRED());
3009 if (error == 0 && backlog != 0)
3010 return (socket_listen(so, backlog, CRED()));
3011
3012 return (error);
3013 }
3014
3015 int
3016 solisten(struct sonode *so, int backlog)
3017 {
3018 return (socket_listen(so, backlog, CRED()));
3019 }
3020
3021 int
3022 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3023 int fflag, int flags)
3024 {
3025 return (socket_connect(so, name, namelen, fflag, flags, CRED()));
3026 }
3027
3028 int
3029 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3030 {
3031 return (socket_recvmsg(so, msg, uiop, CRED()));
3032 }
3033
3034 int
3035 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3036 {
3037 return (socket_sendmsg(so, msg, uiop, CRED()));
3038 }
3039
3040 int
3041 soshutdown(struct sonode *so, int how)
3042 {
3043 return (socket_shutdown(so, how, CRED()));
3044 }
3045
3046 int
3047 sogetsockopt(struct sonode *so, int level, int option_name, void *optval,
3048 socklen_t *optlenp, int flags)
3049 {
3050 return (socket_getsockopt(so, level, option_name, optval, optlenp,
3051 flags, CRED()));
3052 }
3053
3054 int
3055 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval,
3056 t_uscalar_t optlen)
3057 {
3058 return (socket_setsockopt(so, level, option_name, optval, optlen,
3059 CRED()));
3060 }
3061
3062 /*
3063 * Because this is backward compatibility interface it only needs to be
3064 * able to handle the creation of TPI sockfs sockets.
3065 */
3066 struct sonode *
3067 socreate(struct sockparams *sp, int family, int type, int protocol, int version,
3068 int *errorp)
3069 {
3070 struct sonode *so;
3071
3072 ASSERT(sp != NULL);
3073
3074 so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol,
3075 version, SOCKET_SLEEP, errorp, CRED());
3076 if (so == NULL) {
3077 SOCKPARAMS_DEC_REF(sp);
3078 } else {
3079 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) {
3080 /* Cannot fail, only bumps so_count */
3081 (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL);
3082 } else {
3083 socket_destroy(so);
3084 so = NULL;
3085 }
3086 }
3087 return (so);
3088 }