1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2015, Joyent, Inc. All rights reserved.
25 * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
26 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
27 */
28
29 #include <sys/types.h>
30 #include <sys/t_lock.h>
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/buf.h>
34 #include <sys/conf.h>
35 #include <sys/cred.h>
36 #include <sys/kmem.h>
37 #include <sys/sysmacros.h>
38 #include <sys/vfs.h>
39 #include <sys/vnode.h>
40 #include <sys/debug.h>
41 #include <sys/errno.h>
42 #include <sys/time.h>
43 #include <sys/file.h>
44 #include <sys/user.h>
45 #include <sys/stream.h>
46 #include <sys/strsubr.h>
47 #include <sys/strsun.h>
48 #include <sys/sunddi.h>
49 #include <sys/esunddi.h>
50 #include <sys/flock.h>
51 #include <sys/modctl.h>
52 #include <sys/cmn_err.h>
53 #include <sys/vmsystm.h>
54 #include <sys/policy.h>
55 #include <sys/limits.h>
56
57 #include <sys/socket.h>
58 #include <sys/socketvar.h>
59
60 #include <sys/isa_defs.h>
61 #include <sys/inttypes.h>
62 #include <sys/systm.h>
63 #include <sys/cpuvar.h>
64 #include <sys/filio.h>
65 #include <sys/sendfile.h>
66 #include <sys/ddi.h>
67 #include <vm/seg.h>
68 #include <vm/seg_map.h>
69 #include <vm/seg_kpm.h>
70
71 #include <fs/sockfs/nl7c.h>
72 #include <fs/sockfs/sockcommon.h>
73 #include <fs/sockfs/sockfilter_impl.h>
74 #include <fs/sockfs/socktpi.h>
75
76 #ifdef SOCK_TEST
77 int do_useracc = 1; /* Controlled by setting SO_DEBUG to 4 */
78 #else
79 #define do_useracc 1
80 #endif /* SOCK_TEST */
81
82 extern int xnet_truncate_print;
83
84 extern void nl7c_init(void);
85 extern int sockfs_defer_nl7c_init;
86
87 /*
88 * Kernel component of socket creation.
89 *
90 * The socket library determines which version number to use.
91 * First the library calls this with a NULL devpath. If this fails
92 * to find a transport (using solookup) the library will look in /etc/netconfig
93 * for the appropriate transport. If one is found it will pass in the
94 * devpath for the kernel to use.
95 */
96 int
97 so_socket(int family, int type_w_flags, int protocol, char *devpath,
98 int version)
99 {
100 struct sonode *so;
101 vnode_t *vp;
102 struct file *fp;
103 int fd;
104 int error;
105 int type;
106
107 type = type_w_flags & SOCK_TYPE_MASK;
108 type_w_flags &= ~SOCK_TYPE_MASK;
109 if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK))
110 return (set_errno(EINVAL));
111
112 if (devpath != NULL) {
113 char *buf;
114 size_t kdevpathlen = 0;
115
116 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
117 if ((error = copyinstr(devpath, buf,
118 MAXPATHLEN, &kdevpathlen)) != 0) {
119 kmem_free(buf, MAXPATHLEN);
120 return (set_errno(error));
121 }
122 so = socket_create(family, type, protocol, buf, NULL,
123 SOCKET_SLEEP, version, CRED(), &error);
124 kmem_free(buf, MAXPATHLEN);
125 } else {
126 so = socket_create(family, type, protocol, NULL, NULL,
127 SOCKET_SLEEP, version, CRED(), &error);
128 }
129 if (so == NULL)
130 return (set_errno(error));
131
132 /* Allocate a file descriptor for the socket */
133 vp = SOTOV(so);
134 if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) {
135 (void) socket_close(so, 0, CRED());
136 socket_destroy(so);
137 return (set_errno(error));
138 }
139
140 /*
141 * Now fill in the entries that falloc reserved
142 */
143 if (type_w_flags & SOCK_NDELAY) {
144 so->so_state |= SS_NDELAY;
145 fp->f_flag |= FNDELAY;
146 }
147 if (type_w_flags & SOCK_NONBLOCK) {
148 so->so_state |= SS_NONBLOCK;
149 fp->f_flag |= FNONBLOCK;
150 }
151 mutex_exit(&fp->f_tlock);
152 setf(fd, fp);
153 if ((type_w_flags & SOCK_CLOEXEC) != 0) {
154 f_setfd(fd, FD_CLOEXEC);
155 }
156
157 return (fd);
158 }
159
160 /*
161 * Map from a file descriptor to a socket node.
162 * Returns with the file descriptor held i.e. the caller has to
163 * use releasef when done with the file descriptor.
164 */
165 struct sonode *
166 getsonode(int sock, int *errorp, file_t **fpp)
167 {
168 file_t *fp;
169 vnode_t *vp;
170 struct sonode *so;
171
172 if ((fp = getf(sock)) == NULL) {
173 *errorp = EBADF;
174 eprintline(*errorp);
175 return (NULL);
176 }
177 vp = fp->f_vnode;
178 /* Check if it is a socket */
179 if (vp->v_type != VSOCK) {
180 releasef(sock);
181 *errorp = ENOTSOCK;
182 eprintline(*errorp);
183 return (NULL);
184 }
185 /*
186 * Use the stream head to find the real socket vnode.
187 * This is needed when namefs sits above sockfs.
188 */
189 if (vp->v_stream) {
190 ASSERT(vp->v_stream->sd_vnode);
191 vp = vp->v_stream->sd_vnode;
192
193 so = VTOSO(vp);
194 if (so->so_version == SOV_STREAM) {
195 releasef(sock);
196 *errorp = ENOTSOCK;
197 eprintsoline(so, *errorp);
198 return (NULL);
199 }
200 } else {
201 so = VTOSO(vp);
202 }
203 if (fpp)
204 *fpp = fp;
205 return (so);
206 }
207
208 /*
209 * Allocate and copyin a sockaddr.
210 * Ensures NULL termination for AF_UNIX addresses by extending them
211 * with one NULL byte if need be. Verifies that the length is not
212 * excessive to prevent an application from consuming all of kernel
213 * memory. Returns NULL when an error occurred.
214 */
215 static struct sockaddr *
216 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp,
217 int *errorp)
218 {
219 char *faddr;
220 size_t namelen = (size_t)*namelenp;
221
222 ASSERT(namelen != 0);
223 if (namelen > SO_MAXARGSIZE) {
224 *errorp = EINVAL;
225 eprintsoline(so, *errorp);
226 return (NULL);
227 }
228
229 faddr = (char *)kmem_alloc(namelen, KM_SLEEP);
230 if (copyin(name, faddr, namelen)) {
231 kmem_free(faddr, namelen);
232 *errorp = EFAULT;
233 eprintsoline(so, *errorp);
234 return (NULL);
235 }
236
237 /*
238 * Add space for NULL termination if needed.
239 * Do a quick check if the last byte is NUL.
240 */
241 if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') {
242 /* Check if there is any NULL termination */
243 size_t i;
244 int foundnull = 0;
245
246 for (i = sizeof (name->sa_family); i < namelen; i++) {
247 if (faddr[i] == '\0') {
248 foundnull = 1;
249 break;
250 }
251 }
252 if (!foundnull) {
253 /* Add extra byte for NUL padding */
254 char *nfaddr;
255
256 nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP);
257 bcopy(faddr, nfaddr, namelen);
258 kmem_free(faddr, namelen);
259
260 /* NUL terminate */
261 nfaddr[namelen] = '\0';
262 namelen++;
263 ASSERT((socklen_t)namelen == namelen);
264 *namelenp = (socklen_t)namelen;
265 faddr = nfaddr;
266 }
267 }
268 return ((struct sockaddr *)faddr);
269 }
270
271 /*
272 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
273 */
274 static int
275 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
276 socklen_t klen)
277 {
278 if (uaddr != NULL) {
279 if (ulen > klen)
280 ulen = klen;
281
282 if (ulen != 0) {
283 if (copyout(kaddr, uaddr, ulen))
284 return (EFAULT);
285 }
286 } else
287 ulen = 0;
288
289 if (ulenp != NULL) {
290 if (copyout(&ulen, ulenp, sizeof (ulen)))
291 return (EFAULT);
292 }
293 return (0);
294 }
295
296 /*
297 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
298 * If klen is greater than ulen it still uses the non-truncated
299 * klen to update ulenp.
300 */
301 static int
302 copyout_name(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
303 socklen_t klen)
304 {
305 if (uaddr != NULL) {
306 if (ulen >= klen)
307 ulen = klen;
308 else if (ulen != 0 && xnet_truncate_print) {
309 printf("sockfs: truncating copyout of address using "
310 "XNET semantics for pid = %d. Lengths %d, %d\n",
311 curproc->p_pid, klen, ulen);
312 }
313
314 if (ulen != 0) {
315 if (copyout(kaddr, uaddr, ulen))
316 return (EFAULT);
317 } else
318 klen = 0;
319 } else
320 klen = 0;
321
322 if (ulenp != NULL) {
323 if (copyout(&klen, ulenp, sizeof (klen)))
324 return (EFAULT);
325 }
326 return (0);
327 }
328
329 /*
330 * The socketpair() code in libsocket creates two sockets (using
331 * the /etc/netconfig fallback if needed) before calling this routine
332 * to connect the two sockets together.
333 *
334 * For a SOCK_STREAM socketpair a listener is needed - in that case this
335 * routine will create a new file descriptor as part of accepting the
336 * connection. The library socketpair() will check if svs[2] has changed
337 * in which case it will close the changed fd.
338 *
339 * Note that this code could use the TPI feature of accepting the connection
340 * on the listening endpoint. However, that would require significant changes
341 * to soaccept.
342 */
343 int
344 so_socketpair(int sv[2])
345 {
346 int svs[2];
347 struct sonode *so1, *so2;
348 int error;
349 int orig_flags;
350 struct sockaddr_ux *name;
351 size_t namelen;
352 sotpi_info_t *sti1;
353 sotpi_info_t *sti2;
354
355 dprint(1, ("so_socketpair(%p)\n", (void *)sv));
356
357 error = useracc(sv, sizeof (svs), B_WRITE);
358 if (error && do_useracc)
359 return (set_errno(EFAULT));
360
361 if (copyin(sv, svs, sizeof (svs)))
362 return (set_errno(EFAULT));
363
364 if ((so1 = getsonode(svs[0], &error, NULL)) == NULL)
365 return (set_errno(error));
366
367 if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) {
368 releasef(svs[0]);
369 return (set_errno(error));
370 }
371
372 if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) {
373 error = EOPNOTSUPP;
374 goto done;
375 }
376
377 sti1 = SOTOTPI(so1);
378 sti2 = SOTOTPI(so2);
379
380 /*
381 * The code below makes assumptions about the "sockfs" implementation.
382 * So make sure that the correct implementation is really used.
383 */
384 ASSERT(so1->so_ops == &sotpi_sonodeops);
385 ASSERT(so2->so_ops == &sotpi_sonodeops);
386
387 if (so1->so_type == SOCK_DGRAM) {
388 /*
389 * Bind both sockets and connect them with each other.
390 * Need to allocate name/namelen for soconnect.
391 */
392 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED());
393 if (error) {
394 eprintsoline(so1, error);
395 goto done;
396 }
397 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
398 if (error) {
399 eprintsoline(so2, error);
400 goto done;
401 }
402 namelen = sizeof (struct sockaddr_ux);
403 name = kmem_alloc(namelen, KM_SLEEP);
404 name->sou_family = AF_UNIX;
405 name->sou_addr = sti2->sti_ux_laddr;
406 error = socket_connect(so1,
407 (struct sockaddr *)name,
408 (socklen_t)namelen,
409 0, _SOCONNECT_NOXLATE, CRED());
410 if (error) {
411 kmem_free(name, namelen);
412 eprintsoline(so1, error);
413 goto done;
414 }
415 name->sou_addr = sti1->sti_ux_laddr;
416 error = socket_connect(so2,
417 (struct sockaddr *)name,
418 (socklen_t)namelen,
419 0, _SOCONNECT_NOXLATE, CRED());
420 kmem_free(name, namelen);
421 if (error) {
422 eprintsoline(so2, error);
423 goto done;
424 }
425 releasef(svs[0]);
426 releasef(svs[1]);
427 } else {
428 /*
429 * Bind both sockets, with so1 being a listener.
430 * Connect so2 to so1 - nonblocking to avoid waiting for
431 * soaccept to complete.
432 * Accept a connection on so1. Pass out the new fd as sv[0].
433 * The library will detect the changed fd and close
434 * the original one.
435 */
436 struct sonode *nso;
437 struct vnode *nvp;
438 struct file *nfp;
439 int nfd;
440
441 /*
442 * We could simply call socket_listen() here (which would do the
443 * binding automatically) if the code didn't rely on passing
444 * _SOBIND_NOXLATE to the TPI implementation of socket_bind().
445 */
446 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC|
447 _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR,
448 CRED());
449 if (error) {
450 eprintsoline(so1, error);
451 goto done;
452 }
453 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
454 if (error) {
455 eprintsoline(so2, error);
456 goto done;
457 }
458
459 namelen = sizeof (struct sockaddr_ux);
460 name = kmem_alloc(namelen, KM_SLEEP);
461 name->sou_family = AF_UNIX;
462 name->sou_addr = sti1->sti_ux_laddr;
463 error = socket_connect(so2,
464 (struct sockaddr *)name,
465 (socklen_t)namelen,
466 FNONBLOCK, _SOCONNECT_NOXLATE, CRED());
467 kmem_free(name, namelen);
468 if (error) {
469 if (error != EINPROGRESS) {
470 eprintsoline(so2, error); goto done;
471 }
472 }
473
474 error = socket_accept(so1, 0, CRED(), &nso);
475 if (error) {
476 eprintsoline(so1, error);
477 goto done;
478 }
479
480 /* wait for so2 being SS_CONNECTED ignoring signals */
481 mutex_enter(&so2->so_lock);
482 error = sowaitconnected(so2, 0, 1);
483 mutex_exit(&so2->so_lock);
484 if (error != 0) {
485 (void) socket_close(nso, 0, CRED());
486 socket_destroy(nso);
487 eprintsoline(so2, error);
488 goto done;
489 }
490
491 nvp = SOTOV(nso);
492 if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) {
493 (void) socket_close(nso, 0, CRED());
494 socket_destroy(nso);
495 eprintsoline(nso, error);
496 goto done;
497 }
498 /*
499 * copy over FNONBLOCK and FNDELAY flags should they exist
500 */
501 if (so1->so_state & SS_NONBLOCK)
502 nfp->f_flag |= FNONBLOCK;
503 if (so1->so_state & SS_NDELAY)
504 nfp->f_flag |= FNDELAY;
505
506 /*
507 * fill in the entries that falloc reserved
508 */
509 mutex_exit(&nfp->f_tlock);
510 setf(nfd, nfp);
511
512 /*
513 * get the original flags before we release
514 */
515 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0);
516
517 releasef(svs[0]);
518 releasef(svs[1]);
519
520 /*
521 * If FD_CLOEXEC was set on the filedescriptor we're
522 * swapping out, we should set it on the new one too.
523 */
524 if (orig_flags & FD_CLOEXEC) {
525 f_setfd(nfd, FD_CLOEXEC);
526 }
527
528 /*
529 * The socketpair library routine will close the original
530 * svs[0] when this code passes out a different file
531 * descriptor.
532 */
533 svs[0] = nfd;
534
535 if (copyout(svs, sv, sizeof (svs))) {
536 (void) closeandsetf(nfd, NULL);
537 eprintline(EFAULT);
538 return (set_errno(EFAULT));
539 }
540 }
541 return (0);
542
543 done:
544 releasef(svs[0]);
545 releasef(svs[1]);
546 return (set_errno(error));
547 }
548
549 int
550 bind(int sock, struct sockaddr *name, socklen_t namelen, int version)
551 {
552 struct sonode *so;
553 int error;
554
555 dprint(1, ("bind(%d, %p, %d)\n",
556 sock, (void *)name, namelen));
557
558 if ((so = getsonode(sock, &error, NULL)) == NULL)
559 return (set_errno(error));
560
561 /* Allocate and copyin name */
562 /*
563 * X/Open test does not expect EFAULT with NULL name and non-zero
564 * namelen.
565 */
566 if (name != NULL && namelen != 0) {
567 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
568 name = copyin_name(so, name, &namelen, &error);
569 if (name == NULL) {
570 releasef(sock);
571 return (set_errno(error));
572 }
573 } else {
574 name = NULL;
575 namelen = 0;
576 }
577
578 switch (version) {
579 default:
580 error = socket_bind(so, name, namelen, 0, CRED());
581 break;
582 case SOV_XPG4_2:
583 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED());
584 break;
585 case SOV_SOCKBSD:
586 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED());
587 break;
588 }
589 done:
590 releasef(sock);
591 if (name != NULL)
592 kmem_free(name, (size_t)namelen);
593
594 if (error)
595 return (set_errno(error));
596 return (0);
597 }
598
599 /* ARGSUSED2 */
600 int
601 listen(int sock, int backlog, int version)
602 {
603 struct sonode *so;
604 int error;
605
606 dprint(1, ("listen(%d, %d)\n",
607 sock, backlog));
608
609 if ((so = getsonode(sock, &error, NULL)) == NULL)
610 return (set_errno(error));
611
612 error = socket_listen(so, backlog, CRED());
613
614 releasef(sock);
615 if (error)
616 return (set_errno(error));
617 return (0);
618 }
619
620 /*ARGSUSED3*/
621 int
622 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version,
623 int flags)
624 {
625 struct sonode *so;
626 file_t *fp;
627 int error;
628 socklen_t namelen;
629 struct sonode *nso;
630 struct vnode *nvp;
631 struct file *nfp;
632 int nfd;
633 int ssflags;
634 struct sockaddr *addrp;
635 socklen_t addrlen;
636
637 dprint(1, ("accept(%d, %p, %p)\n",
638 sock, (void *)name, (void *)namelenp));
639
640 if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) {
641 return (set_errno(EINVAL));
642 }
643
644 /* Translate SOCK_ flags to their SS_ variant */
645 ssflags = 0;
646 if (flags & SOCK_NONBLOCK)
647 ssflags |= SS_NONBLOCK;
648 if (flags & SOCK_NDELAY)
649 ssflags |= SS_NDELAY;
650
651 if ((so = getsonode(sock, &error, &fp)) == NULL)
652 return (set_errno(error));
653
654 if (name != NULL) {
655 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
656 if (copyin(namelenp, &namelen, sizeof (namelen))) {
657 releasef(sock);
658 return (set_errno(EFAULT));
659 }
660 if (namelen != 0) {
661 error = useracc(name, (size_t)namelen, B_WRITE);
662 if (error && do_useracc) {
663 releasef(sock);
664 return (set_errno(EFAULT));
665 }
666 } else
667 name = NULL;
668 } else {
669 namelen = 0;
670 }
671
672 /*
673 * Allocate the user fd before socket_accept() in order to
674 * catch EMFILE errors before calling socket_accept().
675 */
676 if ((nfd = ufalloc(0)) == -1) {
677 eprintsoline(so, EMFILE);
678 releasef(sock);
679 return (set_errno(EMFILE));
680 }
681 error = socket_accept(so, fp->f_flag, CRED(), &nso);
682 if (error) {
683 setf(nfd, NULL);
684 releasef(sock);
685 return (set_errno(error));
686 }
687
688 nvp = SOTOV(nso);
689
690 ASSERT(MUTEX_NOT_HELD(&nso->so_lock));
691 if (namelen != 0) {
692 addrlen = so->so_max_addr_len;
693 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP);
694
695 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp,
696 &addrlen, B_TRUE, CRED())) == 0) {
697 error = copyout_name(name, namelen, namelenp,
698 addrp, addrlen);
699 } else {
700 ASSERT(error == EINVAL || error == ENOTCONN);
701 error = ECONNABORTED;
702 }
703 kmem_free(addrp, so->so_max_addr_len);
704 }
705
706 if (error) {
707 setf(nfd, NULL);
708 (void) socket_close(nso, 0, CRED());
709 socket_destroy(nso);
710 releasef(sock);
711 return (set_errno(error));
712 }
713 if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) {
714 setf(nfd, NULL);
715 (void) socket_close(nso, 0, CRED());
716 socket_destroy(nso);
717 eprintsoline(so, error);
718 releasef(sock);
719 return (set_errno(error));
720 }
721 /*
722 * fill in the entries that falloc reserved
723 */
724 nfp->f_vnode = nvp;
725 mutex_exit(&nfp->f_tlock);
726 setf(nfd, nfp);
727
728 /*
729 * Act on SOCK_CLOEXEC from flags
730 */
731 if (flags & SOCK_CLOEXEC) {
732 f_setfd(nfd, FD_CLOEXEC);
733 }
734
735 /*
736 * Copy FNDELAY and FNONBLOCK from listener to acceptor
737 * and from ssflags
738 */
739 if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) {
740 uint_t oflag = nfp->f_flag;
741 int arg = 0;
742
743 if ((ssflags | so->so_state) & SS_NONBLOCK)
744 arg |= FNONBLOCK;
745 else if ((ssflags | so->so_state) & SS_NDELAY)
746 arg |= FNDELAY;
747
748 /*
749 * This code is a simplification of the F_SETFL code in fcntl()
750 * Ignore any errors from VOP_SETFL.
751 */
752 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL))
753 != 0) {
754 eprintsoline(so, error);
755 error = 0;
756 } else {
757 mutex_enter(&nfp->f_tlock);
758 nfp->f_flag &= ~FMASK | (FREAD|FWRITE);
759 nfp->f_flag |= arg;
760 mutex_exit(&nfp->f_tlock);
761 }
762 }
763 releasef(sock);
764 return (nfd);
765 }
766
767 int
768 connect(int sock, struct sockaddr *name, socklen_t namelen, int version)
769 {
770 struct sonode *so;
771 file_t *fp;
772 int error;
773
774 dprint(1, ("connect(%d, %p, %d)\n",
775 sock, (void *)name, namelen));
776
777 if ((so = getsonode(sock, &error, &fp)) == NULL)
778 return (set_errno(error));
779
780 /* Allocate and copyin name */
781 if (namelen != 0) {
782 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
783 name = copyin_name(so, name, &namelen, &error);
784 if (name == NULL) {
785 releasef(sock);
786 return (set_errno(error));
787 }
788 } else
789 name = NULL;
790
791 error = socket_connect(so, name, namelen, fp->f_flag,
792 (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED());
793 releasef(sock);
794 if (name)
795 kmem_free(name, (size_t)namelen);
796 if (error)
797 return (set_errno(error));
798 return (0);
799 }
800
801 /*ARGSUSED2*/
802 int
803 shutdown(int sock, int how, int version)
804 {
805 struct sonode *so;
806 int error;
807
808 dprint(1, ("shutdown(%d, %d)\n",
809 sock, how));
810
811 if ((so = getsonode(sock, &error, NULL)) == NULL)
812 return (set_errno(error));
813
814 error = socket_shutdown(so, how, CRED());
815
816 releasef(sock);
817 if (error)
818 return (set_errno(error));
819 return (0);
820 }
821
822 /*
823 * Common receive routine.
824 */
825 static ssize_t
826 recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
827 socklen_t *namelenp, socklen_t *controllenp, int *flagsp)
828 {
829 struct sonode *so;
830 file_t *fp;
831 void *name;
832 socklen_t namelen;
833 void *control;
834 socklen_t controllen;
835 ssize_t len;
836 int error;
837
838 if ((so = getsonode(sock, &error, &fp)) == NULL)
839 return (set_errno(error));
840
841 len = uiop->uio_resid;
842 uiop->uio_fmode = fp->f_flag;
843 uiop->uio_extflg = UIO_COPY_CACHED;
844
845 name = msg->msg_name;
846 namelen = msg->msg_namelen;
847 control = msg->msg_control;
848 controllen = msg->msg_controllen;
849
850 msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL |
851 MSG_DONTWAIT | MSG_XPG4_2);
852
853 error = socket_recvmsg(so, msg, uiop, CRED());
854 if (error) {
855 releasef(sock);
856 return (set_errno(error));
857 }
858 lwp_stat_update(LWP_STAT_MSGRCV, 1);
859 releasef(sock);
860
861 error = copyout_name(name, namelen, namelenp,
862 msg->msg_name, msg->msg_namelen);
863 if (error)
864 goto err;
865
866 if (flagsp != NULL) {
867 /*
868 * Clear internal flag.
869 */
870 msg->msg_flags &= ~MSG_XPG4_2;
871
872 /*
873 * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only
874 * when controllen is zero and there is control data to
875 * copy out.
876 */
877 if (controllen != 0 &&
878 (msg->msg_controllen > controllen || control == NULL)) {
879 dprint(1, ("recvit: CTRUNC %d %d %p\n",
880 msg->msg_controllen, controllen, control));
881
882 msg->msg_flags |= MSG_CTRUNC;
883 }
884 if (copyout(&msg->msg_flags, flagsp,
885 sizeof (msg->msg_flags))) {
886 error = EFAULT;
887 goto err;
888 }
889 }
890 /*
891 * Note: This MUST be done last. There can be no "goto err" after this
892 * point since it could make so_closefds run twice on some part
893 * of the file descriptor array.
894 */
895 if (controllen != 0) {
896 if (!(flags & MSG_XPG4_2)) {
897 /*
898 * Good old msg_accrights can only return a multiple
899 * of 4 bytes.
900 */
901 controllen &= ~((int)sizeof (uint32_t) - 1);
902 }
903 error = copyout_arg(control, controllen, controllenp,
904 msg->msg_control, msg->msg_controllen);
905 if (error)
906 goto err;
907
908 if (msg->msg_controllen > controllen || control == NULL) {
909 if (control == NULL)
910 controllen = 0;
911 so_closefds(msg->msg_control, msg->msg_controllen,
912 !(flags & MSG_XPG4_2), controllen);
913 }
914 }
915 if (msg->msg_namelen != 0)
916 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
917 if (msg->msg_controllen != 0)
918 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
919 return (len - uiop->uio_resid);
920
921 err:
922 /*
923 * If we fail and the control part contains file descriptors
924 * we have to close the fd's.
925 */
926 if (msg->msg_controllen != 0)
927 so_closefds(msg->msg_control, msg->msg_controllen,
928 !(flags & MSG_XPG4_2), 0);
929 if (msg->msg_namelen != 0)
930 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
931 if (msg->msg_controllen != 0)
932 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
933 return (set_errno(error));
934 }
935
936 /*
937 * Native system call
938 */
939 ssize_t
940 recv(int sock, void *buffer, size_t len, int flags)
941 {
942 struct nmsghdr lmsg;
943 struct uio auio;
944 struct iovec aiov[1];
945
946 dprint(1, ("recv(%d, %p, %ld, %d)\n",
947 sock, buffer, len, flags));
948
949 if ((ssize_t)len < 0) {
950 return (set_errno(EINVAL));
951 }
952
953 aiov[0].iov_base = buffer;
954 aiov[0].iov_len = len;
955 auio.uio_loffset = 0;
956 auio.uio_iov = aiov;
957 auio.uio_iovcnt = 1;
958 auio.uio_resid = len;
959 auio.uio_segflg = UIO_USERSPACE;
960 auio.uio_limit = 0;
961
962 lmsg.msg_namelen = 0;
963 lmsg.msg_controllen = 0;
964 lmsg.msg_flags = 0;
965 return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL));
966 }
967
968 ssize_t
969 recvfrom(int sock, void *buffer, size_t len, int flags, struct sockaddr *name,
970 socklen_t *namelenp)
971 {
972 struct nmsghdr lmsg;
973 struct uio auio;
974 struct iovec aiov[1];
975
976 dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n",
977 sock, buffer, len, flags, (void *)name, (void *)namelenp));
978
979 if ((ssize_t)len < 0) {
980 return (set_errno(EINVAL));
981 }
982
983 aiov[0].iov_base = buffer;
984 aiov[0].iov_len = len;
985 auio.uio_loffset = 0;
986 auio.uio_iov = aiov;
987 auio.uio_iovcnt = 1;
988 auio.uio_resid = len;
989 auio.uio_segflg = UIO_USERSPACE;
990 auio.uio_limit = 0;
991
992 lmsg.msg_name = (char *)name;
993 if (namelenp != NULL) {
994 if (copyin(namelenp, &lmsg.msg_namelen,
995 sizeof (lmsg.msg_namelen)))
996 return (set_errno(EFAULT));
997 } else {
998 lmsg.msg_namelen = 0;
999 }
1000 lmsg.msg_controllen = 0;
1001 lmsg.msg_flags = 0;
1002
1003 return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL));
1004 }
1005
1006 /*
1007 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1008 * struct omsghdr or struct nmsghdr.
1009 */
1010 ssize_t
1011 recvmsg(int sock, struct nmsghdr *msg, int flags)
1012 {
1013 STRUCT_DECL(nmsghdr, u_lmsg);
1014 STRUCT_HANDLE(nmsghdr, umsgptr);
1015 struct nmsghdr lmsg;
1016 struct uio auio;
1017 struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1018 ssize_t iovsize = 0;
1019 int iovcnt;
1020 ssize_t len, rval;
1021 int i;
1022 int *flagsp;
1023 model_t model;
1024
1025 dprint(1, ("recvmsg(%d, %p, %d)\n",
1026 sock, (void *)msg, flags));
1027
1028 model = get_udatamodel();
1029 STRUCT_INIT(u_lmsg, model);
1030 STRUCT_SET_HANDLE(umsgptr, model, msg);
1031
1032 if (flags & MSG_XPG4_2) {
1033 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg)))
1034 return (set_errno(EFAULT));
1035 flagsp = STRUCT_FADDR(umsgptr, msg_flags);
1036 } else {
1037 /*
1038 * Assumes that nmsghdr and omsghdr are identically shaped
1039 * except for the added msg_flags field.
1040 */
1041 if (copyin(msg, STRUCT_BUF(u_lmsg),
1042 SIZEOF_STRUCT(omsghdr, model)))
1043 return (set_errno(EFAULT));
1044 STRUCT_FSET(u_lmsg, msg_flags, 0);
1045 flagsp = NULL;
1046 }
1047
1048 /*
1049 * Code below us will kmem_alloc memory and hang it
1050 * off msg_control and msg_name fields. This forces
1051 * us to copy the structure to its native form.
1052 */
1053 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1054 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1055 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1056 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1057 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1058 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1059 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1060
1061 iovcnt = lmsg.msg_iovlen;
1062
1063 if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1064 return (set_errno(EMSGSIZE));
1065 }
1066
1067 if (iovcnt > IOV_MAX_STACK) {
1068 iovsize = iovcnt * sizeof (struct iovec);
1069 aiov = kmem_alloc(iovsize, KM_SLEEP);
1070 }
1071
1072 #ifdef _SYSCALL32_IMPL
1073 /*
1074 * 32-bit callers need to have their iovec expanded, while ensuring
1075 * that they can't move more than 2Gbytes of data in a single call.
1076 */
1077 if (model == DATAMODEL_ILP32) {
1078 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1079 ssize_t iov32size;
1080 ssize32_t count32;
1081
1082 iov32size = iovcnt * sizeof (struct iovec32);
1083 if (iovsize != 0)
1084 aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1085
1086 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1087 if (iovsize != 0) {
1088 kmem_free(aiov32, iov32size);
1089 kmem_free(aiov, iovsize);
1090 }
1091
1092 return (set_errno(EFAULT));
1093 }
1094
1095 count32 = 0;
1096 for (i = 0; i < iovcnt; i++) {
1097 ssize32_t iovlen32;
1098
1099 iovlen32 = aiov32[i].iov_len;
1100 count32 += iovlen32;
1101 if (iovlen32 < 0 || count32 < 0) {
1102 if (iovsize != 0) {
1103 kmem_free(aiov32, iov32size);
1104 kmem_free(aiov, iovsize);
1105 }
1106
1107 return (set_errno(EINVAL));
1108 }
1109
1110 aiov[i].iov_len = iovlen32;
1111 aiov[i].iov_base =
1112 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1113 }
1114
1115 if (iovsize != 0)
1116 kmem_free(aiov32, iov32size);
1117 } else
1118 #endif /* _SYSCALL32_IMPL */
1119 if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) {
1120 if (iovsize != 0)
1121 kmem_free(aiov, iovsize);
1122
1123 return (set_errno(EFAULT));
1124 }
1125 len = 0;
1126 for (i = 0; i < iovcnt; i++) {
1127 ssize_t iovlen = aiov[i].iov_len;
1128 len += iovlen;
1129 if (iovlen < 0 || len < 0) {
1130 if (iovsize != 0)
1131 kmem_free(aiov, iovsize);
1132
1133 return (set_errno(EINVAL));
1134 }
1135 }
1136 auio.uio_loffset = 0;
1137 auio.uio_iov = aiov;
1138 auio.uio_iovcnt = iovcnt;
1139 auio.uio_resid = len;
1140 auio.uio_segflg = UIO_USERSPACE;
1141 auio.uio_limit = 0;
1142
1143 if (lmsg.msg_control != NULL &&
1144 (do_useracc == 0 ||
1145 useracc(lmsg.msg_control, lmsg.msg_controllen,
1146 B_WRITE) != 0)) {
1147 if (iovsize != 0)
1148 kmem_free(aiov, iovsize);
1149
1150 return (set_errno(EFAULT));
1151 }
1152
1153 rval = recvit(sock, &lmsg, &auio, flags,
1154 STRUCT_FADDR(umsgptr, msg_namelen),
1155 STRUCT_FADDR(umsgptr, msg_controllen), flagsp);
1156
1157 if (iovsize != 0)
1158 kmem_free(aiov, iovsize);
1159
1160 return (rval);
1161 }
1162
1163 /*
1164 * Common send function.
1165 */
1166 static ssize_t
1167 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags)
1168 {
1169 struct sonode *so;
1170 file_t *fp;
1171 void *name;
1172 socklen_t namelen;
1173 void *control;
1174 socklen_t controllen;
1175 ssize_t len;
1176 int error;
1177
1178 if ((so = getsonode(sock, &error, &fp)) == NULL)
1179 return (set_errno(error));
1180
1181 uiop->uio_fmode = fp->f_flag;
1182
1183 if (so->so_family == AF_UNIX)
1184 uiop->uio_extflg = UIO_COPY_CACHED;
1185 else
1186 uiop->uio_extflg = UIO_COPY_DEFAULT;
1187
1188 /* Allocate and copyin name and control */
1189 name = msg->msg_name;
1190 namelen = msg->msg_namelen;
1191 if (name != NULL && namelen != 0) {
1192 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1193 name = copyin_name(so,
1194 (struct sockaddr *)name,
1195 &namelen, &error);
1196 if (name == NULL)
1197 goto done3;
1198 /* copyin_name null terminates addresses for AF_UNIX */
1199 msg->msg_namelen = namelen;
1200 msg->msg_name = name;
1201 } else {
1202 msg->msg_name = name = NULL;
1203 msg->msg_namelen = namelen = 0;
1204 }
1205
1206 control = msg->msg_control;
1207 controllen = msg->msg_controllen;
1208 if ((control != NULL) && (controllen != 0)) {
1209 /*
1210 * Verify that the length is not excessive to prevent
1211 * an application from consuming all of kernel memory.
1212 */
1213 if (controllen > SO_MAXARGSIZE) {
1214 error = EINVAL;
1215 goto done2;
1216 }
1217 control = kmem_alloc(controllen, KM_SLEEP);
1218
1219 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1220 if (copyin(msg->msg_control, control, controllen)) {
1221 error = EFAULT;
1222 goto done1;
1223 }
1224 msg->msg_control = control;
1225 } else {
1226 msg->msg_control = control = NULL;
1227 msg->msg_controllen = controllen = 0;
1228 }
1229
1230 len = uiop->uio_resid;
1231 msg->msg_flags = flags;
1232
1233 error = socket_sendmsg(so, msg, uiop, CRED());
1234 done1:
1235 if (control != NULL)
1236 kmem_free(control, controllen);
1237 done2:
1238 if (name != NULL)
1239 kmem_free(name, namelen);
1240 done3:
1241 if (error != 0) {
1242 releasef(sock);
1243 return (set_errno(error));
1244 }
1245 lwp_stat_update(LWP_STAT_MSGSND, 1);
1246 releasef(sock);
1247 return (len - uiop->uio_resid);
1248 }
1249
1250 /*
1251 * Native system call
1252 */
1253 ssize_t
1254 send(int sock, void *buffer, size_t len, int flags)
1255 {
1256 struct nmsghdr lmsg;
1257 struct uio auio;
1258 struct iovec aiov[1];
1259
1260 dprint(1, ("send(%d, %p, %ld, %d)\n",
1261 sock, buffer, len, flags));
1262
1263 if ((ssize_t)len < 0) {
1264 return (set_errno(EINVAL));
1265 }
1266
1267 aiov[0].iov_base = buffer;
1268 aiov[0].iov_len = len;
1269 auio.uio_loffset = 0;
1270 auio.uio_iov = aiov;
1271 auio.uio_iovcnt = 1;
1272 auio.uio_resid = len;
1273 auio.uio_segflg = UIO_USERSPACE;
1274 auio.uio_limit = 0;
1275
1276 lmsg.msg_name = NULL;
1277 lmsg.msg_control = NULL;
1278 if (!(flags & MSG_XPG4_2)) {
1279 /*
1280 * In order to be compatible with the libsocket/sockmod
1281 * implementation we set EOR for all send* calls.
1282 */
1283 flags |= MSG_EOR;
1284 }
1285 return (sendit(sock, &lmsg, &auio, flags));
1286 }
1287
1288 /*
1289 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1290 * struct omsghdr or struct nmsghdr.
1291 */
1292 ssize_t
1293 sendmsg(int sock, struct nmsghdr *msg, int flags)
1294 {
1295 struct nmsghdr lmsg;
1296 STRUCT_DECL(nmsghdr, u_lmsg);
1297 struct uio auio;
1298 struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1299 ssize_t iovsize = 0;
1300 int iovcnt;
1301 ssize_t len, rval;
1302 int i;
1303 model_t model;
1304
1305 dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags));
1306
1307 model = get_udatamodel();
1308 STRUCT_INIT(u_lmsg, model);
1309
1310 if (flags & MSG_XPG4_2) {
1311 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1312 STRUCT_SIZE(u_lmsg)))
1313 return (set_errno(EFAULT));
1314 } else {
1315 /*
1316 * Assumes that nmsghdr and omsghdr are identically shaped
1317 * except for the added msg_flags field.
1318 */
1319 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1320 SIZEOF_STRUCT(omsghdr, model)))
1321 return (set_errno(EFAULT));
1322 /*
1323 * In order to be compatible with the libsocket/sockmod
1324 * implementation we set EOR for all send* calls.
1325 */
1326 flags |= MSG_EOR;
1327 }
1328
1329 /*
1330 * Code below us will kmem_alloc memory and hang it
1331 * off msg_control and msg_name fields. This forces
1332 * us to copy the structure to its native form.
1333 */
1334 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1335 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1336 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1337 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1338 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1339 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1340 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1341
1342 iovcnt = lmsg.msg_iovlen;
1343
1344 if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1345 /*
1346 * Unless this is XPG 4.2 we allow iovcnt == 0 to
1347 * be compatible with SunOS 4.X and 4.4BSD.
1348 */
1349 if (iovcnt != 0 || (flags & MSG_XPG4_2))
1350 return (set_errno(EMSGSIZE));
1351 }
1352
1353 if (iovcnt > IOV_MAX_STACK) {
1354 iovsize = iovcnt * sizeof (struct iovec);
1355 aiov = kmem_alloc(iovsize, KM_SLEEP);
1356 }
1357
1358 #ifdef _SYSCALL32_IMPL
1359 /*
1360 * 32-bit callers need to have their iovec expanded, while ensuring
1361 * that they can't move more than 2Gbytes of data in a single call.
1362 */
1363 if (model == DATAMODEL_ILP32) {
1364 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1365 ssize_t iov32size;
1366 ssize32_t count32;
1367
1368 iov32size = iovcnt * sizeof (struct iovec32);
1369 if (iovsize != 0)
1370 aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1371
1372 if (iovcnt != 0 &&
1373 copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1374 if (iovsize != 0) {
1375 kmem_free(aiov32, iov32size);
1376 kmem_free(aiov, iovsize);
1377 }
1378
1379 return (set_errno(EFAULT));
1380 }
1381
1382 count32 = 0;
1383 for (i = 0; i < iovcnt; i++) {
1384 ssize32_t iovlen32;
1385
1386 iovlen32 = aiov32[i].iov_len;
1387 count32 += iovlen32;
1388 if (iovlen32 < 0 || count32 < 0) {
1389 if (iovsize != 0) {
1390 kmem_free(aiov32, iov32size);
1391 kmem_free(aiov, iovsize);
1392 }
1393
1394 return (set_errno(EINVAL));
1395 }
1396
1397 aiov[i].iov_len = iovlen32;
1398 aiov[i].iov_base =
1399 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1400 }
1401
1402 if (iovsize != 0)
1403 kmem_free(aiov32, iov32size);
1404 } else
1405 #endif /* _SYSCALL32_IMPL */
1406 if (iovcnt != 0 &&
1407 copyin(lmsg.msg_iov, aiov,
1408 (unsigned)iovcnt * sizeof (struct iovec))) {
1409 if (iovsize != 0)
1410 kmem_free(aiov, iovsize);
1411
1412 return (set_errno(EFAULT));
1413 }
1414 len = 0;
1415 for (i = 0; i < iovcnt; i++) {
1416 ssize_t iovlen = aiov[i].iov_len;
1417 len += iovlen;
1418 if (iovlen < 0 || len < 0) {
1419 if (iovsize != 0)
1420 kmem_free(aiov, iovsize);
1421
1422 return (set_errno(EINVAL));
1423 }
1424 }
1425 auio.uio_loffset = 0;
1426 auio.uio_iov = aiov;
1427 auio.uio_iovcnt = iovcnt;
1428 auio.uio_resid = len;
1429 auio.uio_segflg = UIO_USERSPACE;
1430 auio.uio_limit = 0;
1431
1432 rval = sendit(sock, &lmsg, &auio, flags);
1433
1434 if (iovsize != 0)
1435 kmem_free(aiov, iovsize);
1436
1437 return (rval);
1438 }
1439
1440 ssize_t
1441 sendto(int sock, void *buffer, size_t len, int flags,
1442 struct sockaddr *name, socklen_t namelen)
1443 {
1444 struct nmsghdr lmsg;
1445 struct uio auio;
1446 struct iovec aiov[1];
1447
1448 dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n",
1449 sock, buffer, len, flags, (void *)name, namelen));
1450
1451 if ((ssize_t)len < 0) {
1452 return (set_errno(EINVAL));
1453 }
1454
1455 aiov[0].iov_base = buffer;
1456 aiov[0].iov_len = len;
1457 auio.uio_loffset = 0;
1458 auio.uio_iov = aiov;
1459 auio.uio_iovcnt = 1;
1460 auio.uio_resid = len;
1461 auio.uio_segflg = UIO_USERSPACE;
1462 auio.uio_limit = 0;
1463
1464 lmsg.msg_name = (char *)name;
1465 lmsg.msg_namelen = namelen;
1466 lmsg.msg_control = NULL;
1467 if (!(flags & MSG_XPG4_2)) {
1468 /*
1469 * In order to be compatible with the libsocket/sockmod
1470 * implementation we set EOR for all send* calls.
1471 */
1472 flags |= MSG_EOR;
1473 }
1474 return (sendit(sock, &lmsg, &auio, flags));
1475 }
1476
1477 /*ARGSUSED3*/
1478 int
1479 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1480 {
1481 struct sonode *so;
1482 int error;
1483 socklen_t namelen;
1484 socklen_t sock_addrlen;
1485 struct sockaddr *sock_addrp;
1486
1487 dprint(1, ("getpeername(%d, %p, %p)\n",
1488 sock, (void *)name, (void *)namelenp));
1489
1490 if ((so = getsonode(sock, &error, NULL)) == NULL)
1491 goto bad;
1492
1493 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1494 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1495 (name == NULL && namelen != 0)) {
1496 error = EFAULT;
1497 goto rel_out;
1498 }
1499 sock_addrlen = so->so_max_addr_len;
1500 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1501
1502 if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen,
1503 B_FALSE, CRED())) == 0) {
1504 ASSERT(sock_addrlen <= so->so_max_addr_len);
1505 error = copyout_name(name, namelen, namelenp,
1506 (void *)sock_addrp, sock_addrlen);
1507 }
1508 kmem_free(sock_addrp, so->so_max_addr_len);
1509 rel_out:
1510 releasef(sock);
1511 bad: return (error != 0 ? set_errno(error) : 0);
1512 }
1513
1514 /*ARGSUSED3*/
1515 int
1516 getsockname(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1517 {
1518 struct sonode *so;
1519 int error;
1520 socklen_t namelen, sock_addrlen;
1521 struct sockaddr *sock_addrp;
1522
1523 dprint(1, ("getsockname(%d, %p, %p)\n",
1524 sock, (void *)name, (void *)namelenp));
1525
1526 if ((so = getsonode(sock, &error, NULL)) == NULL)
1527 goto bad;
1528
1529 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1530 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1531 (name == NULL && namelen != 0)) {
1532 error = EFAULT;
1533 goto rel_out;
1534 }
1535
1536 sock_addrlen = so->so_max_addr_len;
1537 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1538 if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen,
1539 CRED())) == 0) {
1540 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1541 ASSERT(sock_addrlen <= so->so_max_addr_len);
1542 error = copyout_name(name, namelen, namelenp,
1543 (void *)sock_addrp, sock_addrlen);
1544 }
1545 kmem_free(sock_addrp, so->so_max_addr_len);
1546 rel_out:
1547 releasef(sock);
1548 bad: return (error != 0 ? set_errno(error) : 0);
1549 }
1550
1551 /*ARGSUSED5*/
1552 int
1553 getsockopt(int sock, int level, int option_name, void *option_value,
1554 socklen_t *option_lenp, int version)
1555 {
1556 struct sonode *so;
1557 socklen_t optlen, optlen_res;
1558 void *optval;
1559 int error;
1560
1561 dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n",
1562 sock, level, option_name, option_value, (void *)option_lenp));
1563
1564 if ((so = getsonode(sock, &error, NULL)) == NULL)
1565 return (set_errno(error));
1566
1567 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1568 if (copyin(option_lenp, &optlen, sizeof (optlen))) {
1569 releasef(sock);
1570 return (set_errno(EFAULT));
1571 }
1572 /*
1573 * Verify that the length is not excessive to prevent
1574 * an application from consuming all of kernel memory.
1575 */
1576 if (optlen > SO_MAXARGSIZE) {
1577 error = EINVAL;
1578 releasef(sock);
1579 return (set_errno(error));
1580 }
1581 optval = kmem_alloc(optlen, KM_SLEEP);
1582 optlen_res = optlen;
1583 error = socket_getsockopt(so, level, option_name, optval,
1584 &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2,
1585 CRED());
1586 releasef(sock);
1587 if (error) {
1588 kmem_free(optval, optlen);
1589 return (set_errno(error));
1590 }
1591 error = copyout_arg(option_value, optlen, option_lenp,
1592 optval, optlen_res);
1593 kmem_free(optval, optlen);
1594 if (error)
1595 return (set_errno(error));
1596 return (0);
1597 }
1598
1599 /*ARGSUSED5*/
1600 int
1601 setsockopt(int sock, int level, int option_name, void *option_value,
1602 socklen_t option_len, int version)
1603 {
1604 struct sonode *so;
1605 intptr_t buffer[2];
1606 void *optval = NULL;
1607 int error;
1608
1609 dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n",
1610 sock, level, option_name, option_value, option_len));
1611
1612 if ((so = getsonode(sock, &error, NULL)) == NULL)
1613 return (set_errno(error));
1614
1615 if (option_value != NULL) {
1616 if (option_len != 0) {
1617 /*
1618 * Verify that the length is not excessive to prevent
1619 * an application from consuming all of kernel memory.
1620 */
1621 if (option_len > SO_MAXARGSIZE) {
1622 error = EINVAL;
1623 goto done2;
1624 }
1625 optval = option_len <= sizeof (buffer) ?
1626 &buffer : kmem_alloc((size_t)option_len, KM_SLEEP);
1627 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1628 if (copyin(option_value, optval, (size_t)option_len)) {
1629 error = EFAULT;
1630 goto done1;
1631 }
1632 }
1633 } else
1634 option_len = 0;
1635
1636 error = socket_setsockopt(so, level, option_name, optval,
1637 (t_uscalar_t)option_len, CRED());
1638 done1:
1639 if (optval != buffer)
1640 kmem_free(optval, (size_t)option_len);
1641 done2:
1642 releasef(sock);
1643 if (error)
1644 return (set_errno(error));
1645 return (0);
1646 }
1647
1648 static int
1649 sockconf_add_sock(int family, int type, int protocol, char *name)
1650 {
1651 int error = 0;
1652 char *kdevpath = NULL;
1653 char *kmodule = NULL;
1654 char *buf = NULL;
1655 size_t pathlen = 0;
1656 struct sockparams *sp;
1657
1658 if (name == NULL)
1659 return (EINVAL);
1660 /*
1661 * Copyin the name.
1662 * This also makes it possible to check for too long pathnames.
1663 * Compress the space needed for the name before passing it
1664 * to soconfig - soconfig will store the string until
1665 * the configuration is removed.
1666 */
1667 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1668 if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) {
1669 kmem_free(buf, MAXPATHLEN);
1670 return (error);
1671 }
1672 if (strncmp(buf, "/dev", strlen("/dev")) == 0) {
1673 /* For device */
1674
1675 /*
1676 * Special handling for NCA:
1677 *
1678 * DEV_NCA is never opened even if an application
1679 * requests for AF_NCA. The device opened is instead a
1680 * predefined AF_INET transport (NCA_INET_DEV).
1681 *
1682 * Prior to Volo (PSARC/2007/587) NCA would determine
1683 * the device using a lookup, which worked then because
1684 * all protocols were based on TPI. Since TPI is no
1685 * longer the default, we have to explicitly state
1686 * which device to use.
1687 */
1688 if (strcmp(buf, NCA_DEV) == 0) {
1689 /* only support entry <28, 2, 0> */
1690 if (family != AF_NCA || type != SOCK_STREAM ||
1691 protocol != 0) {
1692 kmem_free(buf, MAXPATHLEN);
1693 return (EINVAL);
1694 }
1695
1696 pathlen = strlen(NCA_INET_DEV) + 1;
1697 kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1698 bcopy(NCA_INET_DEV, kdevpath, pathlen);
1699 kdevpath[pathlen - 1] = '\0';
1700 } else {
1701 kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1702 bcopy(buf, kdevpath, pathlen);
1703 kdevpath[pathlen - 1] = '\0';
1704 }
1705 } else {
1706 /* For socket module */
1707 kmodule = kmem_alloc(pathlen, KM_SLEEP);
1708 bcopy(buf, kmodule, pathlen);
1709 kmodule[pathlen - 1] = '\0';
1710 pathlen = 0;
1711 }
1712 kmem_free(buf, MAXPATHLEN);
1713
1714 /* sockparams_create frees mod name and devpath upon failure */
1715 sp = sockparams_create(family, type, protocol, kmodule,
1716 kdevpath, pathlen, 0, KM_SLEEP, &error);
1717 if (sp != NULL) {
1718 error = sockparams_add(sp);
1719 if (error != 0)
1720 sockparams_destroy(sp);
1721 }
1722
1723 return (error);
1724 }
1725
1726 static int
1727 sockconf_remove_sock(int family, int type, int protocol)
1728 {
1729 return (sockparams_delete(family, type, protocol));
1730 }
1731
1732 static int
1733 sockconfig_remove_filter(const char *uname)
1734 {
1735 char kname[SOF_MAXNAMELEN];
1736 size_t len;
1737 int error;
1738 sof_entry_t *ent;
1739
1740 if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0)
1741 return (error);
1742
1743 ent = sof_entry_remove_by_name(kname);
1744 if (ent == NULL)
1745 return (ENXIO);
1746
1747 mutex_enter(&ent->sofe_lock);
1748 ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED));
1749 if (ent->sofe_refcnt == 0) {
1750 mutex_exit(&ent->sofe_lock);
1751 sof_entry_free(ent);
1752 } else {
1753 /* let the last socket free the filter */
1754 ent->sofe_flags |= SOFEF_CONDEMED;
1755 mutex_exit(&ent->sofe_lock);
1756 }
1757
1758 return (0);
1759 }
1760
1761 static int
1762 sockconfig_add_filter(const char *uname, void *ufilpropp)
1763 {
1764 struct sockconfig_filter_props filprop;
1765 sof_entry_t *ent;
1766 int error;
1767 size_t tuplesz, len;
1768 char hintbuf[SOF_MAXNAMELEN];
1769
1770 ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP);
1771 mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL);
1772
1773 if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN,
1774 &len)) != 0) {
1775 sof_entry_free(ent);
1776 return (error);
1777 }
1778
1779 if (get_udatamodel() == DATAMODEL_NATIVE) {
1780 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) {
1781 sof_entry_free(ent);
1782 return (EFAULT);
1783 }
1784 }
1785 #ifdef _SYSCALL32_IMPL
1786 else {
1787 struct sockconfig_filter_props32 filprop32;
1788
1789 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) {
1790 sof_entry_free(ent);
1791 return (EFAULT);
1792 }
1793 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname;
1794 filprop.sfp_autoattach = filprop32.sfp_autoattach;
1795 filprop.sfp_hint = filprop32.sfp_hint;
1796 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg;
1797 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt;
1798 filprop.sfp_socktuple =
1799 (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple;
1800 }
1801 #endif /* _SYSCALL32_IMPL */
1802
1803 if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname,
1804 sizeof (ent->sofe_modname), &len)) != 0) {
1805 sof_entry_free(ent);
1806 return (error);
1807 }
1808
1809 /*
1810 * A filter must specify at least one socket tuple.
1811 */
1812 if (filprop.sfp_socktuple_cnt == 0 ||
1813 filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) {
1814 sof_entry_free(ent);
1815 return (EINVAL);
1816 }
1817 ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG;
1818 ent->sofe_hint = filprop.sfp_hint;
1819
1820 /*
1821 * Verify the hint, and copy in the hint argument, if necessary.
1822 */
1823 switch (ent->sofe_hint) {
1824 case SOF_HINT_BEFORE:
1825 case SOF_HINT_AFTER:
1826 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf,
1827 sizeof (hintbuf), &len)) != 0) {
1828 sof_entry_free(ent);
1829 return (error);
1830 }
1831 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP);
1832 bcopy(hintbuf, ent->sofe_hintarg, len);
1833 /* FALLTHRU */
1834 case SOF_HINT_TOP:
1835 case SOF_HINT_BOTTOM:
1836 /* hints cannot be used with programmatic filters */
1837 if (ent->sofe_flags & SOFEF_PROG) {
1838 sof_entry_free(ent);
1839 return (EINVAL);
1840 }
1841 break;
1842 case SOF_HINT_NONE:
1843 break;
1844 default:
1845 /* bad hint value */
1846 sof_entry_free(ent);
1847 return (EINVAL);
1848 }
1849
1850 ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt;
1851 tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt;
1852 ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP);
1853
1854 if (get_udatamodel() == DATAMODEL_NATIVE) {
1855 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple,
1856 tuplesz)) {
1857 sof_entry_free(ent);
1858 return (EFAULT);
1859 }
1860 }
1861 #ifdef _SYSCALL32_IMPL
1862 else {
1863 int i;
1864 caddr_t data = (caddr_t)filprop.sfp_socktuple;
1865 sof_socktuple_t *tup = ent->sofe_socktuple;
1866 sof_socktuple32_t tup32;
1867
1868 tup = ent->sofe_socktuple;
1869 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) {
1870 ASSERT(tup < ent->sofe_socktuple + tuplesz);
1871
1872 if (copyin(data, &tup32, sizeof (tup32)) != 0) {
1873 sof_entry_free(ent);
1874 return (EFAULT);
1875 }
1876 tup->sofst_family = tup32.sofst_family;
1877 tup->sofst_type = tup32.sofst_type;
1878 tup->sofst_protocol = tup32.sofst_protocol;
1879
1880 data += sizeof (tup32);
1881 }
1882 }
1883 #endif /* _SYSCALL32_IMPL */
1884
1885 /* Sockets can start using the filter as soon as the filter is added */
1886 if ((error = sof_entry_add(ent)) != 0)
1887 sof_entry_free(ent);
1888
1889 return (error);
1890 }
1891
1892 /*
1893 * Socket configuration system call. It is used to add and remove
1894 * socket types.
1895 */
1896 int
1897 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4)
1898 {
1899 int error = 0;
1900
1901 if (secpolicy_net_config(CRED(), B_FALSE) != 0)
1902 return (set_errno(EPERM));
1903
1904 if (sockfs_defer_nl7c_init) {
1905 nl7c_init();
1906 sockfs_defer_nl7c_init = 0;
1907 }
1908
1909 switch (cmd) {
1910 case SOCKCONFIG_ADD_SOCK:
1911 error = sockconf_add_sock((int)(uintptr_t)arg1,
1912 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4);
1913 break;
1914 case SOCKCONFIG_REMOVE_SOCK:
1915 error = sockconf_remove_sock((int)(uintptr_t)arg1,
1916 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3);
1917 break;
1918 case SOCKCONFIG_ADD_FILTER:
1919 error = sockconfig_add_filter((const char *)arg1, arg2);
1920 break;
1921 case SOCKCONFIG_REMOVE_FILTER:
1922 error = sockconfig_remove_filter((const char *)arg1);
1923 break;
1924 case SOCKCONFIG_GET_SOCKTABLE:
1925 error = sockparams_copyout_socktable((int)(uintptr_t)arg1);
1926 break;
1927 default:
1928 #ifdef DEBUG
1929 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd);
1930 #endif
1931 error = EINVAL;
1932 break;
1933 }
1934
1935 if (error != 0) {
1936 eprintline(error);
1937 return (set_errno(error));
1938 }
1939 return (0);
1940 }
1941
1942
1943 /*
1944 * Sendfile is implemented through two schemes, direct I/O or by
1945 * caching in the filesystem page cache. We cache the input file by
1946 * default and use direct I/O only if sendfile_max_size is set
1947 * appropriately as explained below. Note that this logic is consistent
1948 * with other filesystems where caching is turned on by default
1949 * unless explicitly turned off by using the DIRECTIO ioctl.
1950 *
1951 * We choose a slightly different scheme here. One can turn off
1952 * caching by setting sendfile_max_size to 0. One can also enable
1953 * caching of files <= sendfile_max_size by setting sendfile_max_size
1954 * to an appropriate value. By default sendfile_max_size is set to the
1955 * maximum value so that all files are cached. In future, we may provide
1956 * better interfaces for caching the file.
1957 *
1958 * Sendfile through Direct I/O (Zero copy)
1959 * --------------------------------------
1960 *
1961 * As disks are normally slower than the network, we can't have a
1962 * single thread that reads the disk and writes to the network. We
1963 * need to have parallelism. This is done by having the sendfile
1964 * thread create another thread that reads from the filesystem
1965 * and queues it for network processing. In this scheme, the data
1966 * is never copied anywhere i.e it is zero copy unlike the other
1967 * scheme.
1968 *
1969 * We have a sendfile queue (snfq) where each sendfile
1970 * request (snf_req_t) is queued for processing by a thread. Number
1971 * of threads is dynamically allocated and they exit if they are idling
1972 * beyond a specified amount of time. When each request (snf_req_t) is
1973 * processed by a thread, it produces a number of mblk_t structures to
1974 * be consumed by the sendfile thread. snf_deque and snf_enque are
1975 * used for consuming and producing mblks. Size of the filesystem
1976 * read is determined by the tunable (sendfile_read_size). A single
1977 * mblk holds sendfile_read_size worth of data (except the last
1978 * read of the file) which is sent down as a whole to the network.
1979 * sendfile_read_size is set to 1 MB as this seems to be the optimal
1980 * value for the UFS filesystem backed by a striped storage array.
1981 *
1982 * Synchronisation between read (producer) and write (consumer) threads.
1983 * --------------------------------------------------------------------
1984 *
1985 * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while
1986 * adding and deleting items in this list. Error can happen anytime
1987 * during read or write. There could be unprocessed mblks in the
1988 * sr_ib_XXX list when a read or write error occurs. Whenever error
1989 * is encountered, we need two things to happen :
1990 *
1991 * a) One of the threads need to clean the mblks.
1992 * b) When one thread encounters an error, the other should stop.
1993 *
1994 * For (a), we don't want to penalize the reader thread as it could do
1995 * some useful work processing other requests. For (b), the error can
1996 * be detected by examining sr_read_error or sr_write_error.
1997 * sr_lock protects sr_read_error and sr_write_error. If both reader and
1998 * writer encounters error, we need to report the write error back to
1999 * the application as that's what would have happened if the operations
2000 * were done sequentially. With this in mind, following should work :
2001 *
2002 * - Check for errors before read or write.
2003 * - If the reader encounters error, set the error in sr_read_error.
2004 * Check sr_write_error, if it is set, send cv_signal as it is
2005 * waiting for reader to complete. If it is not set, the writer
2006 * is either running sinking data to the network or blocked
2007 * because of flow control. For handling the latter case, we
2008 * always send a signal. In any case, it will examine sr_read_error
2009 * and return. sr_read_error is marked with SR_READ_DONE to tell
2010 * the writer that the reader is done in all the cases.
2011 * - If the writer encounters error, set the error in sr_write_error.
2012 * The reader thread is either blocked because of flow control or
2013 * running reading data from the disk. For the former, we need to
2014 * wakeup the thread. Again to keep it simple, we always wake up
2015 * the reader thread. Then, wait for the read thread to complete
2016 * if it is not done yet. Cleanup and return.
2017 *
2018 * High and low water marks for the read thread.
2019 * --------------------------------------------
2020 *
2021 * If sendfile() is used to send data over a slow network, we need to
2022 * make sure that the read thread does not produce data at a faster
2023 * rate than the network. This can happen if the disk is faster than
2024 * the network. In such a case, we don't want to build a very large queue.
2025 * But we would still like to get all of the network throughput possible.
2026 * This implies that network should never block waiting for data.
2027 * As there are lot of disk throughput/network throughput combinations
2028 * possible, it is difficult to come up with an accurate number.
2029 * A typical 10K RPM disk has a max seek latency 17ms and rotational
2030 * latency of 3ms for reading a disk block. Thus, the total latency to
2031 * initiate a new read, transfer data from the disk and queue for
2032 * transmission would take about a max of 25ms. Todays max transfer rate
2033 * for network is 100MB/sec. If the thread is blocked because of flow
2034 * control, it would take 25ms to get new data ready for transmission.
2035 * We have to make sure that network is not idling, while we are initiating
2036 * new transfers. So, at 100MB/sec, to keep network busy we would need
2037 * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data.
2038 * We need to pick a high water mark so that the woken up thread would
2039 * do considerable work before blocking again to prevent thrashing. Currently,
2040 * we pick this to be 10 times that of the low water mark.
2041 *
2042 * Sendfile with segmap caching (One copy from page cache to mblks).
2043 * ----------------------------------------------------------------
2044 *
2045 * We use the segmap cache for caching the file, if the size of file
2046 * is <= sendfile_max_size. In this case we don't use threads as VM
2047 * is reasonably fast enough to keep up with the network. If the underlying
2048 * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth
2049 * of data into segmap space, and use the virtual address from segmap
2050 * directly through desballoc() to avoid copy. Once the transport is done
2051 * with the data, the mapping will be released through segmap_release()
2052 * called by the call-back routine.
2053 *
2054 * If zero-copy is not allowed by the transport, we simply call VOP_READ()
2055 * to copy the data from the filesystem into our temporary network buffer.
2056 *
2057 * To disable caching, set sendfile_max_size to 0.
2058 */
2059
2060 uint_t sendfile_read_size = 1024 * 1024;
2061 #define SENDFILE_REQ_LOWAT 3 * 1024 * 1024
2062 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT;
2063 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT;
2064 struct sendfile_stats sf_stats;
2065 struct sendfile_queue *snfq;
2066 clock_t snfq_timeout;
2067 off64_t sendfile_max_size;
2068
2069 static void snf_enque(snf_req_t *, mblk_t *);
2070 static mblk_t *snf_deque(snf_req_t *);
2071
2072 void
2073 sendfile_init(void)
2074 {
2075 snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP);
2076
2077 mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL);
2078 cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL);
2079 snfq->snfq_max_threads = max_ncpus;
2080 snfq_timeout = SNFQ_TIMEOUT;
2081 /* Cache all files by default. */
2082 sendfile_max_size = MAXOFFSET_T;
2083 }
2084
2085 /*
2086 * Queues a mblk_t for network processing.
2087 */
2088 static void
2089 snf_enque(snf_req_t *sr, mblk_t *mp)
2090 {
2091 mp->b_next = NULL;
2092 mutex_enter(&sr->sr_lock);
2093 if (sr->sr_mp_head == NULL) {
2094 sr->sr_mp_head = sr->sr_mp_tail = mp;
2095 cv_signal(&sr->sr_cv);
2096 } else {
2097 sr->sr_mp_tail->b_next = mp;
2098 sr->sr_mp_tail = mp;
2099 }
2100 sr->sr_qlen += MBLKL(mp);
2101 while ((sr->sr_qlen > sr->sr_hiwat) &&
2102 (sr->sr_write_error == 0)) {
2103 sf_stats.ss_full_waits++;
2104 cv_wait(&sr->sr_cv, &sr->sr_lock);
2105 }
2106 mutex_exit(&sr->sr_lock);
2107 }
2108
2109 /*
2110 * De-queues a mblk_t for network processing.
2111 */
2112 static mblk_t *
2113 snf_deque(snf_req_t *sr)
2114 {
2115 mblk_t *mp;
2116
2117 mutex_enter(&sr->sr_lock);
2118 /*
2119 * If we have encountered an error on read or read is
2120 * completed and no more mblks, return NULL.
2121 * We need to check for NULL sr_mp_head also as
2122 * the reads could have completed and there is
2123 * nothing more to come.
2124 */
2125 if (((sr->sr_read_error & ~SR_READ_DONE) != 0) ||
2126 ((sr->sr_read_error & SR_READ_DONE) &&
2127 sr->sr_mp_head == NULL)) {
2128 mutex_exit(&sr->sr_lock);
2129 return (NULL);
2130 }
2131 /*
2132 * To start with neither SR_READ_DONE is marked nor
2133 * the error is set. When we wake up from cv_wait,
2134 * following are the possibilities :
2135 *
2136 * a) sr_read_error is zero and mblks are queued.
2137 * b) sr_read_error is set to SR_READ_DONE
2138 * and mblks are queued.
2139 * c) sr_read_error is set to SR_READ_DONE
2140 * and no mblks.
2141 * d) sr_read_error is set to some error other
2142 * than SR_READ_DONE.
2143 */
2144
2145 while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) {
2146 sf_stats.ss_empty_waits++;
2147 cv_wait(&sr->sr_cv, &sr->sr_lock);
2148 }
2149 /* Handle (a) and (b) first - the normal case. */
2150 if (((sr->sr_read_error & ~SR_READ_DONE) == 0) &&
2151 (sr->sr_mp_head != NULL)) {
2152 mp = sr->sr_mp_head;
2153 sr->sr_mp_head = mp->b_next;
2154 sr->sr_qlen -= MBLKL(mp);
2155 if (sr->sr_qlen < sr->sr_lowat)
2156 cv_signal(&sr->sr_cv);
2157 mutex_exit(&sr->sr_lock);
2158 mp->b_next = NULL;
2159 return (mp);
2160 }
2161 /* Handle (c) and (d). */
2162 mutex_exit(&sr->sr_lock);
2163 return (NULL);
2164 }
2165
2166 /*
2167 * Reads data from the filesystem and queues it for network processing.
2168 */
2169 void
2170 snf_async_read(snf_req_t *sr)
2171 {
2172 size_t iosize;
2173 u_offset_t fileoff;
2174 u_offset_t size;
2175 int ret_size;
2176 int error;
2177 file_t *fp;
2178 mblk_t *mp;
2179 struct vnode *vp;
2180 int extra = 0;
2181 int maxblk = 0;
2182 int wroff = 0;
2183 struct sonode *so;
2184
2185 fp = sr->sr_fp;
2186 size = sr->sr_file_size;
2187 fileoff = sr->sr_file_off;
2188
2189 /*
2190 * Ignore the error for filesystems that doesn't support DIRECTIO.
2191 */
2192 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0,
2193 kcred, NULL, NULL);
2194
2195 vp = sr->sr_vp;
2196 if (vp->v_type == VSOCK) {
2197 stdata_t *stp;
2198
2199 /*
2200 * Get the extra space to insert a header and a trailer.
2201 */
2202 so = VTOSO(vp);
2203 stp = vp->v_stream;
2204 if (stp == NULL) {
2205 wroff = so->so_proto_props.sopp_wroff;
2206 maxblk = so->so_proto_props.sopp_maxblk;
2207 extra = wroff + so->so_proto_props.sopp_tail;
2208 } else {
2209 wroff = (int)(stp->sd_wroff);
2210 maxblk = (int)(stp->sd_maxblk);
2211 extra = wroff + (int)(stp->sd_tail);
2212 }
2213 }
2214
2215 while ((size != 0) && (sr->sr_write_error == 0)) {
2216
2217 iosize = (int)MIN(sr->sr_maxpsz, size);
2218
2219 /*
2220 * Socket filters can limit the mblk size,
2221 * so limit reads to maxblk if there are
2222 * filters present.
2223 */
2224 if (vp->v_type == VSOCK &&
2225 so->so_filter_active > 0 && maxblk != INFPSZ)
2226 iosize = (int)MIN(iosize, maxblk);
2227
2228 if (is_system_labeled()) {
2229 mp = allocb_cred(iosize + extra, CRED(),
2230 curproc->p_pid);
2231 } else {
2232 mp = allocb(iosize + extra, BPRI_MED);
2233 }
2234 if (mp == NULL) {
2235 error = EAGAIN;
2236 break;
2237 }
2238
2239 mp->b_rptr += wroff;
2240
2241 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize);
2242
2243 /* Error or Reached EOF ? */
2244 if ((error != 0) || (ret_size == 0)) {
2245 freeb(mp);
2246 break;
2247 }
2248 mp->b_wptr = mp->b_rptr + ret_size;
2249
2250 snf_enque(sr, mp);
2251 size -= ret_size;
2252 fileoff += ret_size;
2253 }
2254 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0,
2255 kcred, NULL, NULL);
2256 mutex_enter(&sr->sr_lock);
2257 sr->sr_read_error = error;
2258 sr->sr_read_error |= SR_READ_DONE;
2259 cv_signal(&sr->sr_cv);
2260 mutex_exit(&sr->sr_lock);
2261 }
2262
2263 void
2264 snf_async_thread(void)
2265 {
2266 snf_req_t *sr;
2267 callb_cpr_t cprinfo;
2268 clock_t time_left = 1;
2269
2270 CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq");
2271
2272 mutex_enter(&snfq->snfq_lock);
2273 for (;;) {
2274 /*
2275 * If we didn't find a entry, then block until woken up
2276 * again and then look through the queues again.
2277 */
2278 while ((sr = snfq->snfq_req_head) == NULL) {
2279 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2280 if (time_left <= 0) {
2281 snfq->snfq_svc_threads--;
2282 CALLB_CPR_EXIT(&cprinfo);
2283 thread_exit();
2284 /* NOTREACHED */
2285 }
2286 snfq->snfq_idle_cnt++;
2287
2288 time_left = cv_reltimedwait(&snfq->snfq_cv,
2289 &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK);
2290 snfq->snfq_idle_cnt--;
2291
2292 CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock);
2293 }
2294 snfq->snfq_req_head = sr->sr_next;
2295 snfq->snfq_req_cnt--;
2296 mutex_exit(&snfq->snfq_lock);
2297 snf_async_read(sr);
2298 mutex_enter(&snfq->snfq_lock);
2299 }
2300 }
2301
2302
2303 snf_req_t *
2304 create_thread(int operation, struct vnode *vp, file_t *fp,
2305 u_offset_t fileoff, u_offset_t size)
2306 {
2307 snf_req_t *sr;
2308 stdata_t *stp;
2309
2310 sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP);
2311
2312 sr->sr_vp = vp;
2313 sr->sr_fp = fp;
2314 stp = vp->v_stream;
2315
2316 /*
2317 * store sd_qn_maxpsz into sr_maxpsz while we have stream head.
2318 * stream might be closed before thread returns from snf_async_read.
2319 */
2320 if (stp != NULL && stp->sd_qn_maxpsz > 0) {
2321 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz);
2322 } else {
2323 sr->sr_maxpsz = MAXBSIZE;
2324 }
2325
2326 sr->sr_operation = operation;
2327 sr->sr_file_off = fileoff;
2328 sr->sr_file_size = size;
2329 sr->sr_hiwat = sendfile_req_hiwat;
2330 sr->sr_lowat = sendfile_req_lowat;
2331 mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL);
2332 cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL);
2333 /*
2334 * See whether we need another thread for servicing this
2335 * request. If there are already enough requests queued
2336 * for the threads, create one if not exceeding
2337 * snfq_max_threads.
2338 */
2339 mutex_enter(&snfq->snfq_lock);
2340 if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt &&
2341 snfq->snfq_svc_threads < snfq->snfq_max_threads) {
2342 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0,
2343 TS_RUN, minclsyspri);
2344 snfq->snfq_svc_threads++;
2345 }
2346 if (snfq->snfq_req_head == NULL) {
2347 snfq->snfq_req_head = snfq->snfq_req_tail = sr;
2348 cv_signal(&snfq->snfq_cv);
2349 } else {
2350 snfq->snfq_req_tail->sr_next = sr;
2351 snfq->snfq_req_tail = sr;
2352 }
2353 snfq->snfq_req_cnt++;
2354 mutex_exit(&snfq->snfq_lock);
2355 return (sr);
2356 }
2357
2358 int
2359 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size,
2360 ssize_t *count)
2361 {
2362 snf_req_t *sr;
2363 mblk_t *mp;
2364 int iosize;
2365 int error = 0;
2366 short fflag;
2367 struct vnode *vp;
2368 int ksize;
2369 struct nmsghdr msg;
2370
2371 ksize = 0;
2372 *count = 0;
2373 bzero(&msg, sizeof (msg));
2374
2375 vp = fp->f_vnode;
2376 fflag = fp->f_flag;
2377 if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL)
2378 return (EAGAIN);
2379
2380 /*
2381 * We check for read error in snf_deque. It has to check
2382 * for successful READ_DONE and return NULL, and we might
2383 * as well make an additional check there.
2384 */
2385 while ((mp = snf_deque(sr)) != NULL) {
2386
2387 if (ISSIG(curthread, JUSTLOOKING)) {
2388 freeb(mp);
2389 error = EINTR;
2390 break;
2391 }
2392 iosize = MBLKL(mp);
2393
2394 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2395
2396 if (error != 0) {
2397 if (mp != NULL)
2398 freeb(mp);
2399 break;
2400 }
2401 ksize += iosize;
2402 }
2403 *count = ksize;
2404
2405 mutex_enter(&sr->sr_lock);
2406 sr->sr_write_error = error;
2407 /* Look at the big comments on why we cv_signal here. */
2408 cv_signal(&sr->sr_cv);
2409
2410 /* Wait for the reader to complete always. */
2411 while (!(sr->sr_read_error & SR_READ_DONE)) {
2412 cv_wait(&sr->sr_cv, &sr->sr_lock);
2413 }
2414 /* If there is no write error, check for read error. */
2415 if (error == 0)
2416 error = (sr->sr_read_error & ~SR_READ_DONE);
2417
2418 if (error != 0) {
2419 mblk_t *next_mp;
2420
2421 mp = sr->sr_mp_head;
2422 while (mp != NULL) {
2423 next_mp = mp->b_next;
2424 mp->b_next = NULL;
2425 freeb(mp);
2426 mp = next_mp;
2427 }
2428 }
2429 mutex_exit(&sr->sr_lock);
2430 kmem_free(sr, sizeof (snf_req_t));
2431 return (error);
2432 }
2433
2434 /* Maximum no.of pages allocated by vpm for sendfile at a time */
2435 #define SNF_VPMMAXPGS (VPMMAXPGS/2)
2436
2437 /*
2438 * Maximum no.of elements in the list returned by vpm, including
2439 * NULL for the last entry
2440 */
2441 #define SNF_MAXVMAPS (SNF_VPMMAXPGS + 1)
2442
2443 typedef struct {
2444 unsigned int snfv_ref;
2445 frtn_t snfv_frtn;
2446 vnode_t *snfv_vp;
2447 struct vmap snfv_vml[SNF_MAXVMAPS];
2448 } snf_vmap_desbinfo;
2449
2450 typedef struct {
2451 frtn_t snfi_frtn;
2452 caddr_t snfi_base;
2453 uint_t snfi_mapoff;
2454 size_t snfi_len;
2455 vnode_t *snfi_vp;
2456 } snf_smap_desbinfo;
2457
2458 /*
2459 * The callback function used for vpm mapped mblks called when the last ref of
2460 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2461 * can be the driver too due to lazy reclaim.
2462 */
2463 void
2464 snf_vmap_desbfree(snf_vmap_desbinfo *snfv)
2465 {
2466 ASSERT(snfv->snfv_ref != 0);
2467 if (atomic_dec_32_nv(&snfv->snfv_ref) == 0) {
2468 vpm_unmap_pages(snfv->snfv_vml, S_READ);
2469 VN_RELE(snfv->snfv_vp);
2470 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2471 }
2472 }
2473
2474 /*
2475 * The callback function used for segmap'ped mblks called when the last ref of
2476 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2477 * can be the driver too due to lazy reclaim.
2478 */
2479 void
2480 snf_smap_desbfree(snf_smap_desbinfo *snfi)
2481 {
2482 if (! IS_KPM_ADDR(snfi->snfi_base)) {
2483 /*
2484 * We don't need to call segmap_fault(F_SOFTUNLOCK) for
2485 * segmap_kpm as long as the latter never falls back to
2486 * "use_segmap_range". (See segmap_getmapflt().)
2487 *
2488 * Using S_OTHER saves an redundant hat_setref() in
2489 * segmap_unlock()
2490 */
2491 (void) segmap_fault(kas.a_hat, segkmap,
2492 (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base +
2493 snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len,
2494 F_SOFTUNLOCK, S_OTHER);
2495 }
2496 (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED);
2497 VN_RELE(snfi->snfi_vp);
2498 kmem_free(snfi, sizeof (*snfi));
2499 }
2500
2501 /*
2502 * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk.
2503 * When segmap is used, the mblk contains a segmap slot of no more
2504 * than MAXBSIZE.
2505 *
2506 * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained
2507 * in each iteration and sent by socket_sendmblk until an error occurs or
2508 * the requested size has been transferred. An mblk is esballoca'ed from
2509 * each mapped page and a chain of these mblk is sent to the transport layer.
2510 * vpm will be called to unmap the pages when all mblks have been freed by
2511 * free_func.
2512 *
2513 * At the end of the whole sendfile() operation, we wait till the data from
2514 * the last mblk is ack'ed by the transport before returning so that the
2515 * caller of sendfile() can safely modify the file content.
2516 *
2517 * The caller of this function should make sure that total_size does not exceed
2518 * the actual file size of fvp.
2519 */
2520 int
2521 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size,
2522 ssize_t *count, boolean_t nowait)
2523 {
2524 caddr_t base;
2525 int mapoff;
2526 vnode_t *vp;
2527 mblk_t *mp = NULL;
2528 int chain_size;
2529 int error;
2530 clock_t deadlk_wait;
2531 short fflag;
2532 int ksize;
2533 struct vattr va;
2534 boolean_t dowait = B_FALSE;
2535 struct nmsghdr msg;
2536
2537 vp = fp->f_vnode;
2538 fflag = fp->f_flag;
2539 ksize = 0;
2540 bzero(&msg, sizeof (msg));
2541
2542 for (;;) {
2543 if (ISSIG(curthread, JUSTLOOKING)) {
2544 error = EINTR;
2545 break;
2546 }
2547
2548 if (vpm_enable) {
2549 snf_vmap_desbinfo *snfv;
2550 mblk_t *nmp;
2551 int mblk_size;
2552 int maxsize;
2553 int i;
2554
2555 mapoff = fileoff & PAGEOFFSET;
2556 maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size);
2557
2558 snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo),
2559 KM_SLEEP);
2560
2561 /*
2562 * Get vpm mappings for maxsize with read access.
2563 * If the pages aren't available yet, we get
2564 * DEADLK, so wait and try again a little later using
2565 * an increasing wait. We might be here a long time.
2566 *
2567 * If delay_sig returns EINTR, be sure to exit and
2568 * pass it up to the caller.
2569 */
2570 deadlk_wait = 0;
2571 while ((error = vpm_map_pages(fvp, fileoff,
2572 (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml,
2573 SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) {
2574 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2575 if ((error = delay_sig(deadlk_wait)) != 0) {
2576 break;
2577 }
2578 }
2579 if (error != 0) {
2580 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2581 error = (error == EINTR) ? EINTR : EIO;
2582 goto out;
2583 }
2584 snfv->snfv_frtn.free_func = snf_vmap_desbfree;
2585 snfv->snfv_frtn.free_arg = (caddr_t)snfv;
2586
2587 /* Construct the mblk chain from the page mappings */
2588 chain_size = 0;
2589 for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) &&
2590 total_size > 0; i++) {
2591 ASSERT(chain_size < maxsize);
2592 mblk_size = MIN(snfv->snfv_vml[i].vs_len -
2593 mapoff, total_size);
2594 nmp = esballoca(
2595 (uchar_t *)snfv->snfv_vml[i].vs_addr +
2596 mapoff, mblk_size, BPRI_HI,
2597 &snfv->snfv_frtn);
2598
2599 /*
2600 * We return EAGAIN after unmapping the pages
2601 * if we cannot allocate the the head of the
2602 * chain. Otherwise, we continue sending the
2603 * mblks constructed so far.
2604 */
2605 if (nmp == NULL) {
2606 if (i == 0) {
2607 vpm_unmap_pages(snfv->snfv_vml,
2608 S_READ);
2609 kmem_free(snfv,
2610 sizeof (snf_vmap_desbinfo));
2611 error = EAGAIN;
2612 goto out;
2613 }
2614 break;
2615 }
2616 /* Mark this dblk with the zero-copy flag */
2617 nmp->b_datap->db_struioflag |= STRUIO_ZC;
2618 nmp->b_wptr += mblk_size;
2619 chain_size += mblk_size;
2620 fileoff += mblk_size;
2621 total_size -= mblk_size;
2622 snfv->snfv_ref++;
2623 mapoff = 0;
2624 if (i > 0)
2625 linkb(mp, nmp);
2626 else
2627 mp = nmp;
2628 }
2629 VN_HOLD(fvp);
2630 snfv->snfv_vp = fvp;
2631 } else {
2632 /* vpm not supported. fallback to segmap */
2633 snf_smap_desbinfo *snfi;
2634
2635 mapoff = fileoff & MAXBOFFSET;
2636 chain_size = MAXBSIZE - mapoff;
2637 if (chain_size > total_size)
2638 chain_size = total_size;
2639 /*
2640 * we don't forcefault because we'll call
2641 * segmap_fault(F_SOFTLOCK) next.
2642 *
2643 * S_READ will get the ref bit set (by either
2644 * segmap_getmapflt() or segmap_fault()) and page
2645 * shared locked.
2646 */
2647 base = segmap_getmapflt(segkmap, fvp, fileoff,
2648 chain_size, segmap_kpm ? SM_FAULT : 0, S_READ);
2649
2650 snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP);
2651 snfi->snfi_len = (size_t)roundup(mapoff+chain_size,
2652 PAGESIZE)- (mapoff & PAGEMASK);
2653 /*
2654 * We must call segmap_fault() even for segmap_kpm
2655 * because that's how error gets returned.
2656 * (segmap_getmapflt() never fails but segmap_fault()
2657 * does.)
2658 *
2659 * If the pages aren't available yet, we get
2660 * DEADLK, so wait and try again a little later using
2661 * an increasing wait. We might be here a long time.
2662 *
2663 * If delay_sig returns EINTR, be sure to exit and
2664 * pass it up to the caller.
2665 */
2666 deadlk_wait = 0;
2667 while ((error = FC_ERRNO(segmap_fault(kas.a_hat,
2668 segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base +
2669 mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK,
2670 S_READ))) == EDEADLK) {
2671 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2672 if ((error = delay_sig(deadlk_wait)) != 0) {
2673 break;
2674 }
2675 }
2676 if (error != 0) {
2677 (void) segmap_release(segkmap, base, 0);
2678 kmem_free(snfi, sizeof (*snfi));
2679 error = (error == EINTR) ? EINTR : EIO;
2680 goto out;
2681 }
2682 snfi->snfi_frtn.free_func = snf_smap_desbfree;
2683 snfi->snfi_frtn.free_arg = (caddr_t)snfi;
2684 snfi->snfi_base = base;
2685 snfi->snfi_mapoff = mapoff;
2686 mp = esballoca((uchar_t *)base + mapoff, chain_size,
2687 BPRI_HI, &snfi->snfi_frtn);
2688
2689 if (mp == NULL) {
2690 (void) segmap_fault(kas.a_hat, segkmap,
2691 (caddr_t)(uintptr_t)(((uintptr_t)base +
2692 mapoff) & PAGEMASK), snfi->snfi_len,
2693 F_SOFTUNLOCK, S_OTHER);
2694 (void) segmap_release(segkmap, base, 0);
2695 kmem_free(snfi, sizeof (*snfi));
2696 freemsg(mp);
2697 error = EAGAIN;
2698 goto out;
2699 }
2700 VN_HOLD(fvp);
2701 snfi->snfi_vp = fvp;
2702 mp->b_wptr += chain_size;
2703
2704 /* Mark this dblk with the zero-copy flag */
2705 mp->b_datap->db_struioflag |= STRUIO_ZC;
2706 fileoff += chain_size;
2707 total_size -= chain_size;
2708 }
2709
2710 if (total_size == 0 && !nowait) {
2711 ASSERT(!dowait);
2712 dowait = B_TRUE;
2713 mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
2714 }
2715 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2716 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2717 if (error != 0) {
2718 /*
2719 * mp contains the mblks that were not sent by
2720 * socket_sendmblk. Use its size to update *count
2721 */
2722 *count = ksize + (chain_size - msgdsize(mp));
2723 if (mp != NULL)
2724 freemsg(mp);
2725 return (error);
2726 }
2727 ksize += chain_size;
2728 if (total_size == 0)
2729 goto done;
2730
2731 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2732 va.va_mask = AT_SIZE;
2733 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2734 if (error)
2735 break;
2736 /* Read as much as possible. */
2737 if (fileoff >= va.va_size)
2738 break;
2739 if (total_size + fileoff > va.va_size)
2740 total_size = va.va_size - fileoff;
2741 }
2742 out:
2743 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2744 done:
2745 *count = ksize;
2746 if (dowait) {
2747 stdata_t *stp;
2748
2749 stp = vp->v_stream;
2750 if (stp == NULL) {
2751 struct sonode *so;
2752 so = VTOSO(vp);
2753 error = so_zcopy_wait(so);
2754 } else {
2755 mutex_enter(&stp->sd_lock);
2756 while (!(stp->sd_flag & STZCNOTIFY)) {
2757 if (cv_wait_sig(&stp->sd_zcopy_wait,
2758 &stp->sd_lock) == 0) {
2759 error = EINTR;
2760 break;
2761 }
2762 }
2763 stp->sd_flag &= ~STZCNOTIFY;
2764 mutex_exit(&stp->sd_lock);
2765 }
2766 }
2767 return (error);
2768 }
2769
2770 int
2771 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size,
2772 uint_t maxpsz, ssize_t *count)
2773 {
2774 struct vnode *vp;
2775 mblk_t *mp;
2776 int iosize;
2777 int extra = 0;
2778 int error;
2779 short fflag;
2780 int ksize;
2781 int ioflag;
2782 struct uio auio;
2783 struct iovec aiov;
2784 struct vattr va;
2785 int maxblk = 0;
2786 int wroff = 0;
2787 struct sonode *so;
2788 struct nmsghdr msg;
2789
2790 vp = fp->f_vnode;
2791 if (vp->v_type == VSOCK) {
2792 stdata_t *stp;
2793
2794 /*
2795 * Get the extra space to insert a header and a trailer.
2796 */
2797 so = VTOSO(vp);
2798 stp = vp->v_stream;
2799 if (stp == NULL) {
2800 wroff = so->so_proto_props.sopp_wroff;
2801 maxblk = so->so_proto_props.sopp_maxblk;
2802 extra = wroff + so->so_proto_props.sopp_tail;
2803 } else {
2804 wroff = (int)(stp->sd_wroff);
2805 maxblk = (int)(stp->sd_maxblk);
2806 extra = wroff + (int)(stp->sd_tail);
2807 }
2808 }
2809 bzero(&msg, sizeof (msg));
2810 fflag = fp->f_flag;
2811 ksize = 0;
2812 auio.uio_iov = &aiov;
2813 auio.uio_iovcnt = 1;
2814 auio.uio_segflg = UIO_SYSSPACE;
2815 auio.uio_llimit = MAXOFFSET_T;
2816 auio.uio_fmode = fflag;
2817 auio.uio_extflg = UIO_COPY_CACHED;
2818 ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC);
2819 /* If read sync is not asked for, filter sync flags */
2820 if ((ioflag & FRSYNC) == 0)
2821 ioflag &= ~(FSYNC|FDSYNC);
2822 for (;;) {
2823 if (ISSIG(curthread, JUSTLOOKING)) {
2824 error = EINTR;
2825 break;
2826 }
2827 iosize = (int)MIN(maxpsz, size);
2828
2829 /*
2830 * Socket filters can limit the mblk size,
2831 * so limit reads to maxblk if there are
2832 * filters present.
2833 */
2834 if (vp->v_type == VSOCK &&
2835 so->so_filter_active > 0 && maxblk != INFPSZ)
2836 iosize = (int)MIN(iosize, maxblk);
2837
2838 if (is_system_labeled()) {
2839 mp = allocb_cred(iosize + extra, CRED(),
2840 curproc->p_pid);
2841 } else {
2842 mp = allocb(iosize + extra, BPRI_MED);
2843 }
2844 if (mp == NULL) {
2845 error = EAGAIN;
2846 break;
2847 }
2848
2849 mp->b_rptr += wroff;
2850
2851 aiov.iov_base = (caddr_t)mp->b_rptr;
2852 aiov.iov_len = iosize;
2853 auio.uio_loffset = fileoff;
2854 auio.uio_resid = iosize;
2855
2856 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL);
2857 iosize -= auio.uio_resid;
2858
2859 if (error == EINTR && iosize != 0)
2860 error = 0;
2861
2862 if (error != 0 || iosize == 0) {
2863 freeb(mp);
2864 break;
2865 }
2866 mp->b_wptr = mp->b_rptr + iosize;
2867
2868 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2869
2870 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2871
2872 if (error != 0) {
2873 *count = ksize;
2874 if (mp != NULL)
2875 freeb(mp);
2876 return (error);
2877 }
2878 ksize += iosize;
2879 size -= iosize;
2880 if (size == 0)
2881 goto done;
2882
2883 fileoff += iosize;
2884 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2885 va.va_mask = AT_SIZE;
2886 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2887 if (error)
2888 break;
2889 /* Read as much as possible. */
2890 if (fileoff >= va.va_size)
2891 size = 0;
2892 else if (size + fileoff > va.va_size)
2893 size = va.va_size - fileoff;
2894 }
2895 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2896 done:
2897 *count = ksize;
2898 return (error);
2899 }
2900
2901 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
2902 /*
2903 * Largefile support for 32 bit applications only.
2904 */
2905 int
2906 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv,
2907 ssize32_t *count32)
2908 {
2909 ssize32_t sfv_len;
2910 u_offset_t sfv_off, va_size;
2911 struct vnode *vp, *fvp, *realvp;
2912 struct vattr va;
2913 stdata_t *stp;
2914 ssize_t count = 0;
2915 int error = 0;
2916 boolean_t dozcopy = B_FALSE;
2917 uint_t maxpsz;
2918
2919 sfv_len = (ssize32_t)sfv->sfv_len;
2920 if (sfv_len < 0) {
2921 error = EINVAL;
2922 goto out;
2923 }
2924
2925 if (sfv_len == 0) goto out;
2926
2927 sfv_off = (u_offset_t)sfv->sfv_off;
2928
2929 /* Same checks as in pread */
2930 if (sfv_off > MAXOFFSET_T) {
2931 error = EINVAL;
2932 goto out;
2933 }
2934 if (sfv_off + sfv_len > MAXOFFSET_T)
2935 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off);
2936
2937 /*
2938 * There are no more checks on sfv_len. So, we cast it to
2939 * u_offset_t and share the snf_direct_io/snf_cache code between
2940 * 32 bit and 64 bit.
2941 *
2942 * TODO: should do nbl_need_check() like read()?
2943 */
2944 if (sfv_len > sendfile_max_size) {
2945 sf_stats.ss_file_not_cached++;
2946 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len,
2947 &count);
2948 goto out;
2949 }
2950 fvp = rfp->f_vnode;
2951 if (VOP_REALVP(fvp, &realvp, NULL) == 0)
2952 fvp = realvp;
2953 /*
2954 * Grab the lock as a reader to prevent the file size
2955 * from changing underneath.
2956 */
2957 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2958 va.va_mask = AT_SIZE;
2959 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2960 va_size = va.va_size;
2961 if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) {
2962 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2963 goto out;
2964 }
2965 /* Read as much as possible. */
2966 if (sfv_off + sfv_len > va_size)
2967 sfv_len = va_size - sfv_off;
2968
2969 vp = fp->f_vnode;
2970 stp = vp->v_stream;
2971 /*
2972 * When the NOWAIT flag is not set, we enable zero-copy only if the
2973 * transfer size is large enough. This prevents performance loss
2974 * when the caller sends the file piece by piece.
2975 */
2976 if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) ||
2977 (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) &&
2978 !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) {
2979 uint_t copyflag;
2980 copyflag = stp != NULL ? stp->sd_copyflag :
2981 VTOSO(vp)->so_proto_props.sopp_zcopyflag;
2982 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) {
2983 int on = 1;
2984
2985 if (socket_setsockopt(VTOSO(vp), SOL_SOCKET,
2986 SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0)
2987 dozcopy = B_TRUE;
2988 } else {
2989 dozcopy = copyflag & STZCVMSAFE;
2990 }
2991 }
2992 if (dozcopy) {
2993 sf_stats.ss_file_segmap++;
2994 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2995 &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0));
2996 } else {
2997 if (vp->v_type == VSOCK && stp == NULL) {
2998 sonode_t *so = VTOSO(vp);
2999 maxpsz = so->so_proto_props.sopp_maxpsz;
3000 } else if (stp != NULL) {
3001 maxpsz = stp->sd_qn_maxpsz;
3002 } else {
3003 maxpsz = maxphys;
3004 }
3005
3006 if (maxpsz == INFPSZ)
3007 maxpsz = maxphys;
3008 else
3009 maxpsz = roundup(maxpsz, MAXBSIZE);
3010 sf_stats.ss_file_cached++;
3011 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len,
3012 maxpsz, &count);
3013 }
3014 out:
3015 releasef(sfv->sfv_fd);
3016 *count32 = (ssize32_t)count;
3017 return (error);
3018 }
3019 #endif
3020
3021 #ifdef _SYSCALL32_IMPL
3022 /*
3023 * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a
3024 * ssize_t rather than ssize32_t; see the comments above read32 for details.
3025 */
3026
3027 ssize_t
3028 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3029 {
3030 return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3031 }
3032
3033 ssize_t
3034 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3035 caddr32_t name, caddr32_t namelenp)
3036 {
3037 return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3038 (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp));
3039 }
3040
3041 ssize_t
3042 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3043 {
3044 return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3045 }
3046
3047 ssize_t
3048 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3049 caddr32_t name, socklen_t namelen)
3050 {
3051 return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3052 (void *)(uintptr_t)name, namelen));
3053 }
3054 #endif /* _SYSCALL32_IMPL */
3055
3056 /*
3057 * Function wrappers (mostly around the sonode switch) for
3058 * backward compatibility.
3059 */
3060
3061 int
3062 soaccept(struct sonode *so, int fflag, struct sonode **nsop)
3063 {
3064 return (socket_accept(so, fflag, CRED(), nsop));
3065 }
3066
3067 int
3068 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3069 int backlog, int flags)
3070 {
3071 int error;
3072
3073 error = socket_bind(so, name, namelen, flags, CRED());
3074 if (error == 0 && backlog != 0)
3075 return (socket_listen(so, backlog, CRED()));
3076
3077 return (error);
3078 }
3079
3080 int
3081 solisten(struct sonode *so, int backlog)
3082 {
3083 return (socket_listen(so, backlog, CRED()));
3084 }
3085
3086 int
3087 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3088 int fflag, int flags)
3089 {
3090 return (socket_connect(so, name, namelen, fflag, flags, CRED()));
3091 }
3092
3093 int
3094 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3095 {
3096 return (socket_recvmsg(so, msg, uiop, CRED()));
3097 }
3098
3099 int
3100 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3101 {
3102 return (socket_sendmsg(so, msg, uiop, CRED()));
3103 }
3104
3105 int
3106 soshutdown(struct sonode *so, int how)
3107 {
3108 return (socket_shutdown(so, how, CRED()));
3109 }
3110
3111 int
3112 sogetsockopt(struct sonode *so, int level, int option_name, void *optval,
3113 socklen_t *optlenp, int flags)
3114 {
3115 return (socket_getsockopt(so, level, option_name, optval, optlenp,
3116 flags, CRED()));
3117 }
3118
3119 int
3120 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval,
3121 t_uscalar_t optlen)
3122 {
3123 return (socket_setsockopt(so, level, option_name, optval, optlen,
3124 CRED()));
3125 }
3126
3127 /*
3128 * Because this is backward compatibility interface it only needs to be
3129 * able to handle the creation of TPI sockfs sockets.
3130 */
3131 struct sonode *
3132 socreate(struct sockparams *sp, int family, int type, int protocol, int version,
3133 int *errorp)
3134 {
3135 struct sonode *so;
3136
3137 ASSERT(sp != NULL);
3138
3139 so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol,
3140 version, SOCKET_SLEEP, errorp, CRED());
3141 if (so == NULL) {
3142 SOCKPARAMS_DEC_REF(sp);
3143 } else {
3144 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) {
3145 /* Cannot fail, only bumps so_count */
3146 (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL);
3147 } else {
3148 socket_destroy(so);
3149 so = NULL;
3150 }
3151 }
3152 return (so);
3153 }