1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2015, Joyent, Inc.  All rights reserved.
  25  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
  26  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
  27  */
  28 
  29 #include <sys/types.h>
  30 #include <sys/t_lock.h>
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/buf.h>
  34 #include <sys/conf.h>
  35 #include <sys/cred.h>
  36 #include <sys/kmem.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/vfs.h>
  39 #include <sys/vnode.h>
  40 #include <sys/debug.h>
  41 #include <sys/errno.h>
  42 #include <sys/time.h>
  43 #include <sys/file.h>
  44 #include <sys/user.h>
  45 #include <sys/stream.h>
  46 #include <sys/strsubr.h>
  47 #include <sys/strsun.h>
  48 #include <sys/sunddi.h>
  49 #include <sys/esunddi.h>
  50 #include <sys/flock.h>
  51 #include <sys/modctl.h>
  52 #include <sys/cmn_err.h>
  53 #include <sys/vmsystm.h>
  54 #include <sys/policy.h>
  55 #include <sys/limits.h>
  56 
  57 #include <sys/socket.h>
  58 #include <sys/socketvar.h>
  59 
  60 #include <sys/isa_defs.h>
  61 #include <sys/inttypes.h>
  62 #include <sys/systm.h>
  63 #include <sys/cpuvar.h>
  64 #include <sys/filio.h>
  65 #include <sys/sendfile.h>
  66 #include <sys/ddi.h>
  67 #include <vm/seg.h>
  68 #include <vm/seg_map.h>
  69 #include <vm/seg_kpm.h>
  70 
  71 #include <fs/sockfs/nl7c.h>
  72 #include <fs/sockfs/sockcommon.h>
  73 #include <fs/sockfs/sockfilter_impl.h>
  74 #include <fs/sockfs/socktpi.h>
  75 
  76 #ifdef SOCK_TEST
  77 int do_useracc = 1;             /* Controlled by setting SO_DEBUG to 4 */
  78 #else
  79 #define do_useracc      1
  80 #endif /* SOCK_TEST */
  81 
  82 extern int      xnet_truncate_print;
  83 
  84 extern void     nl7c_init(void);
  85 extern int      sockfs_defer_nl7c_init;
  86 
  87 /*
  88  * Kernel component of socket creation.
  89  *
  90  * The socket library determines which version number to use.
  91  * First the library calls this with a NULL devpath. If this fails
  92  * to find a transport (using solookup) the library will look in /etc/netconfig
  93  * for the appropriate transport. If one is found it will pass in the
  94  * devpath for the kernel to use.
  95  */
  96 int
  97 so_socket(int family, int type_w_flags, int protocol, char *devpath,
  98     int version)
  99 {
 100         struct sonode *so;
 101         vnode_t *vp;
 102         struct file *fp;
 103         int fd;
 104         int error;
 105         int type;
 106 
 107         type = type_w_flags & SOCK_TYPE_MASK;
 108         type_w_flags &= ~SOCK_TYPE_MASK;
 109         if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK))
 110                 return (set_errno(EINVAL));
 111 
 112         if (devpath != NULL) {
 113                 char *buf;
 114                 size_t kdevpathlen = 0;
 115 
 116                 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 117                 if ((error = copyinstr(devpath, buf,
 118                     MAXPATHLEN, &kdevpathlen)) != 0) {
 119                         kmem_free(buf, MAXPATHLEN);
 120                         return (set_errno(error));
 121                 }
 122                 so = socket_create(family, type, protocol, buf, NULL,
 123                     SOCKET_SLEEP, version, CRED(), &error);
 124                 kmem_free(buf, MAXPATHLEN);
 125         } else {
 126                 so = socket_create(family, type, protocol, NULL, NULL,
 127                     SOCKET_SLEEP, version, CRED(), &error);
 128         }
 129         if (so == NULL)
 130                 return (set_errno(error));
 131 
 132         /* Allocate a file descriptor for the socket */
 133         vp = SOTOV(so);
 134         if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) {
 135                 (void) socket_close(so, 0, CRED());
 136                 socket_destroy(so);
 137                 return (set_errno(error));
 138         }
 139 
 140         /*
 141          * Now fill in the entries that falloc reserved
 142          */
 143         if (type_w_flags & SOCK_NDELAY) {
 144                 so->so_state |= SS_NDELAY;
 145                 fp->f_flag |= FNDELAY;
 146         }
 147         if (type_w_flags & SOCK_NONBLOCK) {
 148                 so->so_state |= SS_NONBLOCK;
 149                 fp->f_flag |= FNONBLOCK;
 150         }
 151         mutex_exit(&fp->f_tlock);
 152         setf(fd, fp);
 153         if ((type_w_flags & SOCK_CLOEXEC) != 0) {
 154                 f_setfd(fd, FD_CLOEXEC);
 155         }
 156 
 157         return (fd);
 158 }
 159 
 160 /*
 161  * Map from a file descriptor to a socket node.
 162  * Returns with the file descriptor held i.e. the caller has to
 163  * use releasef when done with the file descriptor.
 164  */
 165 struct sonode *
 166 getsonode(int sock, int *errorp, file_t **fpp)
 167 {
 168         file_t *fp;
 169         vnode_t *vp;
 170         struct sonode *so;
 171 
 172         if ((fp = getf(sock)) == NULL) {
 173                 *errorp = EBADF;
 174                 eprintline(*errorp);
 175                 return (NULL);
 176         }
 177         vp = fp->f_vnode;
 178         /* Check if it is a socket */
 179         if (vp->v_type != VSOCK) {
 180                 releasef(sock);
 181                 *errorp = ENOTSOCK;
 182                 eprintline(*errorp);
 183                 return (NULL);
 184         }
 185         /*
 186          * Use the stream head to find the real socket vnode.
 187          * This is needed when namefs sits above sockfs.
 188          */
 189         if (vp->v_stream) {
 190                 ASSERT(vp->v_stream->sd_vnode);
 191                 vp = vp->v_stream->sd_vnode;
 192 
 193                 so = VTOSO(vp);
 194                 if (so->so_version == SOV_STREAM) {
 195                         releasef(sock);
 196                         *errorp = ENOTSOCK;
 197                         eprintsoline(so, *errorp);
 198                         return (NULL);
 199                 }
 200         } else {
 201                 so = VTOSO(vp);
 202         }
 203         if (fpp)
 204                 *fpp = fp;
 205         return (so);
 206 }
 207 
 208 /*
 209  * Allocate and copyin a sockaddr.
 210  * Ensures NULL termination for AF_UNIX addresses by extending them
 211  * with one NULL byte if need be. Verifies that the length is not
 212  * excessive to prevent an application from consuming all of kernel
 213  * memory. Returns NULL when an error occurred.
 214  */
 215 static struct sockaddr *
 216 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp,
 217     int *errorp)
 218 {
 219         char    *faddr;
 220         size_t  namelen = (size_t)*namelenp;
 221 
 222         ASSERT(namelen != 0);
 223         if (namelen > SO_MAXARGSIZE) {
 224                 *errorp = EINVAL;
 225                 eprintsoline(so, *errorp);
 226                 return (NULL);
 227         }
 228 
 229         faddr = (char *)kmem_alloc(namelen, KM_SLEEP);
 230         if (copyin(name, faddr, namelen)) {
 231                 kmem_free(faddr, namelen);
 232                 *errorp = EFAULT;
 233                 eprintsoline(so, *errorp);
 234                 return (NULL);
 235         }
 236 
 237         /*
 238          * Add space for NULL termination if needed.
 239          * Do a quick check if the last byte is NUL.
 240          */
 241         if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') {
 242                 /* Check if there is any NULL termination */
 243                 size_t  i;
 244                 int foundnull = 0;
 245 
 246                 for (i = sizeof (name->sa_family); i < namelen; i++) {
 247                         if (faddr[i] == '\0') {
 248                                 foundnull = 1;
 249                                 break;
 250                         }
 251                 }
 252                 if (!foundnull) {
 253                         /* Add extra byte for NUL padding */
 254                         char *nfaddr;
 255 
 256                         nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP);
 257                         bcopy(faddr, nfaddr, namelen);
 258                         kmem_free(faddr, namelen);
 259 
 260                         /* NUL terminate */
 261                         nfaddr[namelen] = '\0';
 262                         namelen++;
 263                         ASSERT((socklen_t)namelen == namelen);
 264                         *namelenp = (socklen_t)namelen;
 265                         faddr = nfaddr;
 266                 }
 267         }
 268         return ((struct sockaddr *)faddr);
 269 }
 270 
 271 /*
 272  * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
 273  */
 274 static int
 275 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
 276     socklen_t klen)
 277 {
 278         if (uaddr != NULL) {
 279                 if (ulen > klen)
 280                         ulen = klen;
 281 
 282                 if (ulen != 0) {
 283                         if (copyout(kaddr, uaddr, ulen))
 284                                 return (EFAULT);
 285                 }
 286         } else
 287                 ulen = 0;
 288 
 289         if (ulenp != NULL) {
 290                 if (copyout(&ulen, ulenp, sizeof (ulen)))
 291                         return (EFAULT);
 292         }
 293         return (0);
 294 }
 295 
 296 /*
 297  * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
 298  * If klen is greater than ulen it still uses the non-truncated
 299  * klen to update ulenp.
 300  */
 301 static int
 302 copyout_name(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
 303     socklen_t klen)
 304 {
 305         if (uaddr != NULL) {
 306                 if (ulen >= klen)
 307                         ulen = klen;
 308                 else if (ulen != 0 && xnet_truncate_print) {
 309                         printf("sockfs: truncating copyout of address using "
 310                             "XNET semantics for pid = %d. Lengths %d, %d\n",
 311                             curproc->p_pid, klen, ulen);
 312                 }
 313 
 314                 if (ulen != 0) {
 315                         if (copyout(kaddr, uaddr, ulen))
 316                                 return (EFAULT);
 317                 } else
 318                         klen = 0;
 319         } else
 320                 klen = 0;
 321 
 322         if (ulenp != NULL) {
 323                 if (copyout(&klen, ulenp, sizeof (klen)))
 324                         return (EFAULT);
 325         }
 326         return (0);
 327 }
 328 
 329 /*
 330  * The socketpair() code in libsocket creates two sockets (using
 331  * the /etc/netconfig fallback if needed) before calling this routine
 332  * to connect the two sockets together.
 333  *
 334  * For a SOCK_STREAM socketpair a listener is needed - in that case this
 335  * routine will create a new file descriptor as part of accepting the
 336  * connection. The library socketpair() will check if svs[2] has changed
 337  * in which case it will close the changed fd.
 338  *
 339  * Note that this code could use the TPI feature of accepting the connection
 340  * on the listening endpoint. However, that would require significant changes
 341  * to soaccept.
 342  */
 343 int
 344 so_socketpair(int sv[2])
 345 {
 346         int svs[2];
 347         struct sonode *so1, *so2;
 348         int error;
 349         int orig_flags;
 350         struct sockaddr_ux *name;
 351         size_t namelen;
 352         sotpi_info_t *sti1;
 353         sotpi_info_t *sti2;
 354 
 355         dprint(1, ("so_socketpair(%p)\n", (void *)sv));
 356 
 357         error = useracc(sv, sizeof (svs), B_WRITE);
 358         if (error && do_useracc)
 359                 return (set_errno(EFAULT));
 360 
 361         if (copyin(sv, svs, sizeof (svs)))
 362                 return (set_errno(EFAULT));
 363 
 364         if ((so1 = getsonode(svs[0], &error, NULL)) == NULL)
 365                 return (set_errno(error));
 366 
 367         if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) {
 368                 releasef(svs[0]);
 369                 return (set_errno(error));
 370         }
 371 
 372         if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) {
 373                 error = EOPNOTSUPP;
 374                 goto done;
 375         }
 376 
 377         sti1 = SOTOTPI(so1);
 378         sti2 = SOTOTPI(so2);
 379 
 380         /*
 381          * The code below makes assumptions about the "sockfs" implementation.
 382          * So make sure that the correct implementation is really used.
 383          */
 384         ASSERT(so1->so_ops == &sotpi_sonodeops);
 385         ASSERT(so2->so_ops == &sotpi_sonodeops);
 386 
 387         if (so1->so_type == SOCK_DGRAM) {
 388                 /*
 389                  * Bind both sockets and connect them with each other.
 390                  * Need to allocate name/namelen for soconnect.
 391                  */
 392                 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED());
 393                 if (error) {
 394                         eprintsoline(so1, error);
 395                         goto done;
 396                 }
 397                 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
 398                 if (error) {
 399                         eprintsoline(so2, error);
 400                         goto done;
 401                 }
 402                 namelen = sizeof (struct sockaddr_ux);
 403                 name = kmem_alloc(namelen, KM_SLEEP);
 404                 name->sou_family = AF_UNIX;
 405                 name->sou_addr = sti2->sti_ux_laddr;
 406                 error = socket_connect(so1,
 407                     (struct sockaddr *)name,
 408                     (socklen_t)namelen,
 409                     0, _SOCONNECT_NOXLATE, CRED());
 410                 if (error) {
 411                         kmem_free(name, namelen);
 412                         eprintsoline(so1, error);
 413                         goto done;
 414                 }
 415                 name->sou_addr = sti1->sti_ux_laddr;
 416                 error = socket_connect(so2,
 417                     (struct sockaddr *)name,
 418                     (socklen_t)namelen,
 419                     0, _SOCONNECT_NOXLATE, CRED());
 420                 kmem_free(name, namelen);
 421                 if (error) {
 422                         eprintsoline(so2, error);
 423                         goto done;
 424                 }
 425                 releasef(svs[0]);
 426                 releasef(svs[1]);
 427         } else {
 428                 /*
 429                  * Bind both sockets, with so1 being a listener.
 430                  * Connect so2 to so1 - nonblocking to avoid waiting for
 431                  * soaccept to complete.
 432                  * Accept a connection on so1. Pass out the new fd as sv[0].
 433                  * The library will detect the changed fd and close
 434                  * the original one.
 435                  */
 436                 struct sonode *nso;
 437                 struct vnode *nvp;
 438                 struct file *nfp;
 439                 int nfd;
 440 
 441                 /*
 442                  * We could simply call socket_listen() here (which would do the
 443                  * binding automatically) if the code didn't rely on passing
 444                  * _SOBIND_NOXLATE to the TPI implementation of socket_bind().
 445                  */
 446                 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC|
 447                     _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR,
 448                     CRED());
 449                 if (error) {
 450                         eprintsoline(so1, error);
 451                         goto done;
 452                 }
 453                 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
 454                 if (error) {
 455                         eprintsoline(so2, error);
 456                         goto done;
 457                 }
 458 
 459                 namelen = sizeof (struct sockaddr_ux);
 460                 name = kmem_alloc(namelen, KM_SLEEP);
 461                 name->sou_family = AF_UNIX;
 462                 name->sou_addr = sti1->sti_ux_laddr;
 463                 error = socket_connect(so2,
 464                     (struct sockaddr *)name,
 465                     (socklen_t)namelen,
 466                     FNONBLOCK, _SOCONNECT_NOXLATE, CRED());
 467                 kmem_free(name, namelen);
 468                 if (error) {
 469                         if (error != EINPROGRESS) {
 470                                 eprintsoline(so2, error); goto done;
 471                         }
 472                 }
 473 
 474                 error = socket_accept(so1, 0, CRED(), &nso);
 475                 if (error) {
 476                         eprintsoline(so1, error);
 477                         goto done;
 478                 }
 479 
 480                 /* wait for so2 being SS_CONNECTED ignoring signals */
 481                 mutex_enter(&so2->so_lock);
 482                 error = sowaitconnected(so2, 0, 1);
 483                 mutex_exit(&so2->so_lock);
 484                 if (error != 0) {
 485                         (void) socket_close(nso, 0, CRED());
 486                         socket_destroy(nso);
 487                         eprintsoline(so2, error);
 488                         goto done;
 489                 }
 490 
 491                 nvp = SOTOV(nso);
 492                 if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) {
 493                         (void) socket_close(nso, 0, CRED());
 494                         socket_destroy(nso);
 495                         eprintsoline(nso, error);
 496                         goto done;
 497                 }
 498                 /*
 499                  * copy over FNONBLOCK and FNDELAY flags should they exist
 500                  */
 501                 if (so1->so_state & SS_NONBLOCK)
 502                         nfp->f_flag |= FNONBLOCK;
 503                 if (so1->so_state & SS_NDELAY)
 504                         nfp->f_flag |= FNDELAY;
 505 
 506                 /*
 507                  * fill in the entries that falloc reserved
 508                  */
 509                 mutex_exit(&nfp->f_tlock);
 510                 setf(nfd, nfp);
 511 
 512                 /*
 513                  * get the original flags before we release
 514                  */
 515                 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0);
 516 
 517                 releasef(svs[0]);
 518                 releasef(svs[1]);
 519 
 520                 /*
 521                  * If FD_CLOEXEC was set on the filedescriptor we're
 522                  * swapping out, we should set it on the new one too.
 523                  */
 524                 if (orig_flags & FD_CLOEXEC) {
 525                         f_setfd(nfd, FD_CLOEXEC);
 526                 }
 527 
 528                 /*
 529                  * The socketpair library routine will close the original
 530                  * svs[0] when this code passes out a different file
 531                  * descriptor.
 532                  */
 533                 svs[0] = nfd;
 534 
 535                 if (copyout(svs, sv, sizeof (svs))) {
 536                         (void) closeandsetf(nfd, NULL);
 537                         eprintline(EFAULT);
 538                         return (set_errno(EFAULT));
 539                 }
 540         }
 541         return (0);
 542 
 543 done:
 544         releasef(svs[0]);
 545         releasef(svs[1]);
 546         return (set_errno(error));
 547 }
 548 
 549 int
 550 bind(int sock, struct sockaddr *name, socklen_t namelen, int version)
 551 {
 552         struct sonode *so;
 553         int error;
 554 
 555         dprint(1, ("bind(%d, %p, %d)\n",
 556             sock, (void *)name, namelen));
 557 
 558         if ((so = getsonode(sock, &error, NULL)) == NULL)
 559                 return (set_errno(error));
 560 
 561         /* Allocate and copyin name */
 562         /*
 563          * X/Open test does not expect EFAULT with NULL name and non-zero
 564          * namelen.
 565          */
 566         if (name != NULL && namelen != 0) {
 567                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 568                 name = copyin_name(so, name, &namelen, &error);
 569                 if (name == NULL) {
 570                         releasef(sock);
 571                         return (set_errno(error));
 572                 }
 573         } else {
 574                 name = NULL;
 575                 namelen = 0;
 576         }
 577 
 578         switch (version) {
 579         default:
 580                 error = socket_bind(so, name, namelen, 0, CRED());
 581                 break;
 582         case SOV_XPG4_2:
 583                 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED());
 584                 break;
 585         case SOV_SOCKBSD:
 586                 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED());
 587                 break;
 588         }
 589 done:
 590         releasef(sock);
 591         if (name != NULL)
 592                 kmem_free(name, (size_t)namelen);
 593 
 594         if (error)
 595                 return (set_errno(error));
 596         return (0);
 597 }
 598 
 599 /* ARGSUSED2 */
 600 int
 601 listen(int sock, int backlog, int version)
 602 {
 603         struct sonode *so;
 604         int error;
 605 
 606         dprint(1, ("listen(%d, %d)\n",
 607             sock, backlog));
 608 
 609         if ((so = getsonode(sock, &error, NULL)) == NULL)
 610                 return (set_errno(error));
 611 
 612         error = socket_listen(so, backlog, CRED());
 613 
 614         releasef(sock);
 615         if (error)
 616                 return (set_errno(error));
 617         return (0);
 618 }
 619 
 620 /*ARGSUSED3*/
 621 int
 622 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version,
 623     int flags)
 624 {
 625         struct sonode *so;
 626         file_t *fp;
 627         int error;
 628         socklen_t namelen;
 629         struct sonode *nso;
 630         struct vnode *nvp;
 631         struct file *nfp;
 632         int nfd;
 633         int ssflags;
 634         struct sockaddr *addrp;
 635         socklen_t addrlen;
 636 
 637         dprint(1, ("accept(%d, %p, %p)\n",
 638             sock, (void *)name, (void *)namelenp));
 639 
 640         if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) {
 641                 return (set_errno(EINVAL));
 642         }
 643 
 644         /* Translate SOCK_ flags to their SS_ variant */
 645         ssflags = 0;
 646         if (flags & SOCK_NONBLOCK)
 647                 ssflags |= SS_NONBLOCK;
 648         if (flags & SOCK_NDELAY)
 649                 ssflags |= SS_NDELAY;
 650 
 651         if ((so = getsonode(sock, &error, &fp)) == NULL)
 652                 return (set_errno(error));
 653 
 654         if (name != NULL) {
 655                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 656                 if (copyin(namelenp, &namelen, sizeof (namelen))) {
 657                         releasef(sock);
 658                         return (set_errno(EFAULT));
 659                 }
 660                 if (namelen != 0) {
 661                         error = useracc(name, (size_t)namelen, B_WRITE);
 662                         if (error && do_useracc) {
 663                                 releasef(sock);
 664                                 return (set_errno(EFAULT));
 665                         }
 666                 } else
 667                         name = NULL;
 668         } else {
 669                 namelen = 0;
 670         }
 671 
 672         /*
 673          * Allocate the user fd before socket_accept() in order to
 674          * catch EMFILE errors before calling socket_accept().
 675          */
 676         if ((nfd = ufalloc(0)) == -1) {
 677                 eprintsoline(so, EMFILE);
 678                 releasef(sock);
 679                 return (set_errno(EMFILE));
 680         }
 681         error = socket_accept(so, fp->f_flag, CRED(), &nso);
 682         if (error) {
 683                 setf(nfd, NULL);
 684                 releasef(sock);
 685                 return (set_errno(error));
 686         }
 687 
 688         nvp = SOTOV(nso);
 689 
 690         ASSERT(MUTEX_NOT_HELD(&nso->so_lock));
 691         if (namelen != 0) {
 692                 addrlen = so->so_max_addr_len;
 693                 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP);
 694 
 695                 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp,
 696                     &addrlen, B_TRUE, CRED())) == 0) {
 697                         error = copyout_name(name, namelen, namelenp,
 698                             addrp, addrlen);
 699                 } else {
 700                         ASSERT(error == EINVAL || error == ENOTCONN);
 701                         error = ECONNABORTED;
 702                 }
 703                 kmem_free(addrp, so->so_max_addr_len);
 704         }
 705 
 706         if (error) {
 707                 setf(nfd, NULL);
 708                 (void) socket_close(nso, 0, CRED());
 709                 socket_destroy(nso);
 710                 releasef(sock);
 711                 return (set_errno(error));
 712         }
 713         if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) {
 714                 setf(nfd, NULL);
 715                 (void) socket_close(nso, 0, CRED());
 716                 socket_destroy(nso);
 717                 eprintsoline(so, error);
 718                 releasef(sock);
 719                 return (set_errno(error));
 720         }
 721         /*
 722          * fill in the entries that falloc reserved
 723          */
 724         nfp->f_vnode = nvp;
 725         mutex_exit(&nfp->f_tlock);
 726         setf(nfd, nfp);
 727 
 728         /*
 729          * Act on SOCK_CLOEXEC from flags
 730          */
 731         if (flags & SOCK_CLOEXEC) {
 732                 f_setfd(nfd, FD_CLOEXEC);
 733         }
 734 
 735         /*
 736          * Copy FNDELAY and FNONBLOCK from listener to acceptor
 737          * and from ssflags
 738          */
 739         if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) {
 740                 uint_t oflag = nfp->f_flag;
 741                 int arg = 0;
 742 
 743                 if ((ssflags | so->so_state) & SS_NONBLOCK)
 744                         arg |= FNONBLOCK;
 745                 else if ((ssflags | so->so_state) & SS_NDELAY)
 746                         arg |= FNDELAY;
 747 
 748                 /*
 749                  * This code is a simplification of the F_SETFL code in fcntl()
 750                  * Ignore any errors from VOP_SETFL.
 751                  */
 752                 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL))
 753                     != 0) {
 754                         eprintsoline(so, error);
 755                         error = 0;
 756                 } else {
 757                         mutex_enter(&nfp->f_tlock);
 758                         nfp->f_flag &= ~FMASK | (FREAD|FWRITE);
 759                         nfp->f_flag |= arg;
 760                         mutex_exit(&nfp->f_tlock);
 761                 }
 762         }
 763         releasef(sock);
 764         return (nfd);
 765 }
 766 
 767 int
 768 connect(int sock, struct sockaddr *name, socklen_t namelen, int version)
 769 {
 770         struct sonode *so;
 771         file_t *fp;
 772         int error;
 773 
 774         dprint(1, ("connect(%d, %p, %d)\n",
 775             sock, (void *)name, namelen));
 776 
 777         if ((so = getsonode(sock, &error, &fp)) == NULL)
 778                 return (set_errno(error));
 779 
 780         /* Allocate and copyin name */
 781         if (namelen != 0) {
 782                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 783                 name = copyin_name(so, name, &namelen, &error);
 784                 if (name == NULL) {
 785                         releasef(sock);
 786                         return (set_errno(error));
 787                 }
 788         } else
 789                 name = NULL;
 790 
 791         error = socket_connect(so, name, namelen, fp->f_flag,
 792             (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED());
 793         releasef(sock);
 794         if (name)
 795                 kmem_free(name, (size_t)namelen);
 796         if (error)
 797                 return (set_errno(error));
 798         return (0);
 799 }
 800 
 801 /*ARGSUSED2*/
 802 int
 803 shutdown(int sock, int how, int version)
 804 {
 805         struct sonode *so;
 806         int error;
 807 
 808         dprint(1, ("shutdown(%d, %d)\n",
 809             sock, how));
 810 
 811         if ((so = getsonode(sock, &error, NULL)) == NULL)
 812                 return (set_errno(error));
 813 
 814         error = socket_shutdown(so, how, CRED());
 815 
 816         releasef(sock);
 817         if (error)
 818                 return (set_errno(error));
 819         return (0);
 820 }
 821 
 822 /*
 823  * Common receive routine.
 824  */
 825 static ssize_t
 826 recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
 827     socklen_t *namelenp, socklen_t *controllenp, int *flagsp)
 828 {
 829         struct sonode *so;
 830         file_t *fp;
 831         void *name;
 832         socklen_t namelen;
 833         void *control;
 834         socklen_t controllen;
 835         ssize_t len;
 836         int error;
 837 
 838         if ((so = getsonode(sock, &error, &fp)) == NULL)
 839                 return (set_errno(error));
 840 
 841         len = uiop->uio_resid;
 842         uiop->uio_fmode = fp->f_flag;
 843         uiop->uio_extflg = UIO_COPY_CACHED;
 844 
 845         name = msg->msg_name;
 846         namelen = msg->msg_namelen;
 847         control = msg->msg_control;
 848         controllen = msg->msg_controllen;
 849 
 850         msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL |
 851             MSG_DONTWAIT | MSG_XPG4_2);
 852 
 853         error = socket_recvmsg(so, msg, uiop, CRED());
 854         if (error) {
 855                 releasef(sock);
 856                 return (set_errno(error));
 857         }
 858         lwp_stat_update(LWP_STAT_MSGRCV, 1);
 859         releasef(sock);
 860 
 861         error = copyout_name(name, namelen, namelenp,
 862             msg->msg_name, msg->msg_namelen);
 863         if (error)
 864                 goto err;
 865 
 866         if (flagsp != NULL) {
 867                 /*
 868                  * Clear internal flag.
 869                  */
 870                 msg->msg_flags &= ~MSG_XPG4_2;
 871 
 872                 /*
 873                  * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only
 874                  * when controllen is zero and there is control data to
 875                  * copy out.
 876                  */
 877                 if (controllen != 0 &&
 878                     (msg->msg_controllen > controllen || control == NULL)) {
 879                         dprint(1, ("recvit: CTRUNC %d %d %p\n",
 880                             msg->msg_controllen, controllen, control));
 881 
 882                         msg->msg_flags |= MSG_CTRUNC;
 883                 }
 884                 if (copyout(&msg->msg_flags, flagsp,
 885                     sizeof (msg->msg_flags))) {
 886                         error = EFAULT;
 887                         goto err;
 888                 }
 889         }
 890         /*
 891          * Note: This MUST be done last. There can be no "goto err" after this
 892          * point since it could make so_closefds run twice on some part
 893          * of the file descriptor array.
 894          */
 895         if (controllen != 0) {
 896                 if (!(flags & MSG_XPG4_2)) {
 897                         /*
 898                          * Good old msg_accrights can only return a multiple
 899                          * of 4 bytes.
 900                          */
 901                         controllen &= ~((int)sizeof (uint32_t) - 1);
 902                 }
 903                 error = copyout_arg(control, controllen, controllenp,
 904                     msg->msg_control, msg->msg_controllen);
 905                 if (error)
 906                         goto err;
 907 
 908                 if (msg->msg_controllen > controllen || control == NULL) {
 909                         if (control == NULL)
 910                                 controllen = 0;
 911                         so_closefds(msg->msg_control, msg->msg_controllen,
 912                             !(flags & MSG_XPG4_2), controllen);
 913                 }
 914         }
 915         if (msg->msg_namelen != 0)
 916                 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
 917         if (msg->msg_controllen != 0)
 918                 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
 919         return (len - uiop->uio_resid);
 920 
 921 err:
 922         /*
 923          * If we fail and the control part contains file descriptors
 924          * we have to close the fd's.
 925          */
 926         if (msg->msg_controllen != 0)
 927                 so_closefds(msg->msg_control, msg->msg_controllen,
 928                     !(flags & MSG_XPG4_2), 0);
 929         if (msg->msg_namelen != 0)
 930                 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
 931         if (msg->msg_controllen != 0)
 932                 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
 933         return (set_errno(error));
 934 }
 935 
 936 /*
 937  * Native system call
 938  */
 939 ssize_t
 940 recv(int sock, void *buffer, size_t len, int flags)
 941 {
 942         struct nmsghdr lmsg;
 943         struct uio auio;
 944         struct iovec aiov[1];
 945 
 946         dprint(1, ("recv(%d, %p, %ld, %d)\n",
 947             sock, buffer, len, flags));
 948 
 949         if ((ssize_t)len < 0) {
 950                 return (set_errno(EINVAL));
 951         }
 952 
 953         aiov[0].iov_base = buffer;
 954         aiov[0].iov_len = len;
 955         auio.uio_loffset = 0;
 956         auio.uio_iov = aiov;
 957         auio.uio_iovcnt = 1;
 958         auio.uio_resid = len;
 959         auio.uio_segflg = UIO_USERSPACE;
 960         auio.uio_limit = 0;
 961 
 962         lmsg.msg_namelen = 0;
 963         lmsg.msg_controllen = 0;
 964         lmsg.msg_flags = 0;
 965         return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL));
 966 }
 967 
 968 ssize_t
 969 recvfrom(int sock, void *buffer, size_t len, int flags, struct sockaddr *name,
 970     socklen_t *namelenp)
 971 {
 972         struct nmsghdr lmsg;
 973         struct uio auio;
 974         struct iovec aiov[1];
 975 
 976         dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n",
 977             sock, buffer, len, flags, (void *)name, (void *)namelenp));
 978 
 979         if ((ssize_t)len < 0) {
 980                 return (set_errno(EINVAL));
 981         }
 982 
 983         aiov[0].iov_base = buffer;
 984         aiov[0].iov_len = len;
 985         auio.uio_loffset = 0;
 986         auio.uio_iov = aiov;
 987         auio.uio_iovcnt = 1;
 988         auio.uio_resid = len;
 989         auio.uio_segflg = UIO_USERSPACE;
 990         auio.uio_limit = 0;
 991 
 992         lmsg.msg_name = (char *)name;
 993         if (namelenp != NULL) {
 994                 if (copyin(namelenp, &lmsg.msg_namelen,
 995                     sizeof (lmsg.msg_namelen)))
 996                         return (set_errno(EFAULT));
 997         } else {
 998                 lmsg.msg_namelen = 0;
 999         }
1000         lmsg.msg_controllen = 0;
1001         lmsg.msg_flags = 0;
1002 
1003         return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL));
1004 }
1005 
1006 /*
1007  * Uses the MSG_XPG4_2 flag to determine if the caller is using
1008  * struct omsghdr or struct nmsghdr.
1009  */
1010 ssize_t
1011 recvmsg(int sock, struct nmsghdr *msg, int flags)
1012 {
1013         STRUCT_DECL(nmsghdr, u_lmsg);
1014         STRUCT_HANDLE(nmsghdr, umsgptr);
1015         struct nmsghdr lmsg;
1016         struct uio auio;
1017         struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1018         ssize_t iovsize = 0;
1019         int iovcnt;
1020         ssize_t len, rval;
1021         int i;
1022         int *flagsp;
1023         model_t model;
1024 
1025         dprint(1, ("recvmsg(%d, %p, %d)\n",
1026             sock, (void *)msg, flags));
1027 
1028         model = get_udatamodel();
1029         STRUCT_INIT(u_lmsg, model);
1030         STRUCT_SET_HANDLE(umsgptr, model, msg);
1031 
1032         if (flags & MSG_XPG4_2) {
1033                 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg)))
1034                         return (set_errno(EFAULT));
1035                 flagsp = STRUCT_FADDR(umsgptr, msg_flags);
1036         } else {
1037                 /*
1038                  * Assumes that nmsghdr and omsghdr are identically shaped
1039                  * except for the added msg_flags field.
1040                  */
1041                 if (copyin(msg, STRUCT_BUF(u_lmsg),
1042                     SIZEOF_STRUCT(omsghdr, model)))
1043                         return (set_errno(EFAULT));
1044                 STRUCT_FSET(u_lmsg, msg_flags, 0);
1045                 flagsp = NULL;
1046         }
1047 
1048         /*
1049          * Code below us will kmem_alloc memory and hang it
1050          * off msg_control and msg_name fields. This forces
1051          * us to copy the structure to its native form.
1052          */
1053         lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1054         lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1055         lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1056         lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1057         lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1058         lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1059         lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1060 
1061         iovcnt = lmsg.msg_iovlen;
1062 
1063         if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1064                 return (set_errno(EMSGSIZE));
1065         }
1066 
1067         if (iovcnt > IOV_MAX_STACK) {
1068                 iovsize = iovcnt * sizeof (struct iovec);
1069                 aiov = kmem_alloc(iovsize, KM_SLEEP);
1070         }
1071 
1072 #ifdef _SYSCALL32_IMPL
1073         /*
1074          * 32-bit callers need to have their iovec expanded, while ensuring
1075          * that they can't move more than 2Gbytes of data in a single call.
1076          */
1077         if (model == DATAMODEL_ILP32) {
1078                 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1079                 ssize_t iov32size;
1080                 ssize32_t count32;
1081 
1082                 iov32size = iovcnt * sizeof (struct iovec32);
1083                 if (iovsize != 0)
1084                         aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1085 
1086                 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1087                         if (iovsize != 0) {
1088                                 kmem_free(aiov32, iov32size);
1089                                 kmem_free(aiov, iovsize);
1090                         }
1091 
1092                         return (set_errno(EFAULT));
1093                 }
1094 
1095                 count32 = 0;
1096                 for (i = 0; i < iovcnt; i++) {
1097                         ssize32_t iovlen32;
1098 
1099                         iovlen32 = aiov32[i].iov_len;
1100                         count32 += iovlen32;
1101                         if (iovlen32 < 0 || count32 < 0) {
1102                                 if (iovsize != 0) {
1103                                         kmem_free(aiov32, iov32size);
1104                                         kmem_free(aiov, iovsize);
1105                                 }
1106 
1107                                 return (set_errno(EINVAL));
1108                         }
1109 
1110                         aiov[i].iov_len = iovlen32;
1111                         aiov[i].iov_base =
1112                             (caddr_t)(uintptr_t)aiov32[i].iov_base;
1113                 }
1114 
1115                 if (iovsize != 0)
1116                         kmem_free(aiov32, iov32size);
1117         } else
1118 #endif /* _SYSCALL32_IMPL */
1119         if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) {
1120                 if (iovsize != 0)
1121                         kmem_free(aiov, iovsize);
1122 
1123                 return (set_errno(EFAULT));
1124         }
1125         len = 0;
1126         for (i = 0; i < iovcnt; i++) {
1127                 ssize_t iovlen = aiov[i].iov_len;
1128                 len += iovlen;
1129                 if (iovlen < 0 || len < 0) {
1130                         if (iovsize != 0)
1131                                 kmem_free(aiov, iovsize);
1132 
1133                         return (set_errno(EINVAL));
1134                 }
1135         }
1136         auio.uio_loffset = 0;
1137         auio.uio_iov = aiov;
1138         auio.uio_iovcnt = iovcnt;
1139         auio.uio_resid = len;
1140         auio.uio_segflg = UIO_USERSPACE;
1141         auio.uio_limit = 0;
1142 
1143         if (lmsg.msg_control != NULL &&
1144             (do_useracc == 0 ||
1145             useracc(lmsg.msg_control, lmsg.msg_controllen,
1146             B_WRITE) != 0)) {
1147                 if (iovsize != 0)
1148                         kmem_free(aiov, iovsize);
1149 
1150                 return (set_errno(EFAULT));
1151         }
1152 
1153         rval = recvit(sock, &lmsg, &auio, flags,
1154             STRUCT_FADDR(umsgptr, msg_namelen),
1155             STRUCT_FADDR(umsgptr, msg_controllen), flagsp);
1156 
1157         if (iovsize != 0)
1158                 kmem_free(aiov, iovsize);
1159 
1160         return (rval);
1161 }
1162 
1163 /*
1164  * Common send function.
1165  */
1166 static ssize_t
1167 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags)
1168 {
1169         struct sonode *so;
1170         file_t *fp;
1171         void *name;
1172         socklen_t namelen;
1173         void *control;
1174         socklen_t controllen;
1175         ssize_t len;
1176         int error;
1177 
1178         if ((so = getsonode(sock, &error, &fp)) == NULL)
1179                 return (set_errno(error));
1180 
1181         uiop->uio_fmode = fp->f_flag;
1182 
1183         if (so->so_family == AF_UNIX)
1184                 uiop->uio_extflg = UIO_COPY_CACHED;
1185         else
1186                 uiop->uio_extflg = UIO_COPY_DEFAULT;
1187 
1188         /* Allocate and copyin name and control */
1189         name = msg->msg_name;
1190         namelen = msg->msg_namelen;
1191         if (name != NULL && namelen != 0) {
1192                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1193                 name = copyin_name(so,
1194                     (struct sockaddr *)name,
1195                     &namelen, &error);
1196                 if (name == NULL)
1197                         goto done3;
1198                 /* copyin_name null terminates addresses for AF_UNIX */
1199                 msg->msg_namelen = namelen;
1200                 msg->msg_name = name;
1201         } else {
1202                 msg->msg_name = name = NULL;
1203                 msg->msg_namelen = namelen = 0;
1204         }
1205 
1206         control = msg->msg_control;
1207         controllen = msg->msg_controllen;
1208         if ((control != NULL) && (controllen != 0)) {
1209                 /*
1210                  * Verify that the length is not excessive to prevent
1211                  * an application from consuming all of kernel memory.
1212                  */
1213                 if (controllen > SO_MAXARGSIZE) {
1214                         error = EINVAL;
1215                         goto done2;
1216                 }
1217                 control = kmem_alloc(controllen, KM_SLEEP);
1218 
1219                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1220                 if (copyin(msg->msg_control, control, controllen)) {
1221                         error = EFAULT;
1222                         goto done1;
1223                 }
1224                 msg->msg_control = control;
1225         } else {
1226                 msg->msg_control = control = NULL;
1227                 msg->msg_controllen = controllen = 0;
1228         }
1229 
1230         len = uiop->uio_resid;
1231         msg->msg_flags = flags;
1232 
1233         error = socket_sendmsg(so, msg, uiop, CRED());
1234 done1:
1235         if (control != NULL)
1236                 kmem_free(control, controllen);
1237 done2:
1238         if (name != NULL)
1239                 kmem_free(name, namelen);
1240 done3:
1241         if (error != 0) {
1242                 releasef(sock);
1243                 return (set_errno(error));
1244         }
1245         lwp_stat_update(LWP_STAT_MSGSND, 1);
1246         releasef(sock);
1247         return (len - uiop->uio_resid);
1248 }
1249 
1250 /*
1251  * Native system call
1252  */
1253 ssize_t
1254 send(int sock, void *buffer, size_t len, int flags)
1255 {
1256         struct nmsghdr lmsg;
1257         struct uio auio;
1258         struct iovec aiov[1];
1259 
1260         dprint(1, ("send(%d, %p, %ld, %d)\n",
1261             sock, buffer, len, flags));
1262 
1263         if ((ssize_t)len < 0) {
1264                 return (set_errno(EINVAL));
1265         }
1266 
1267         aiov[0].iov_base = buffer;
1268         aiov[0].iov_len = len;
1269         auio.uio_loffset = 0;
1270         auio.uio_iov = aiov;
1271         auio.uio_iovcnt = 1;
1272         auio.uio_resid = len;
1273         auio.uio_segflg = UIO_USERSPACE;
1274         auio.uio_limit = 0;
1275 
1276         lmsg.msg_name = NULL;
1277         lmsg.msg_control = NULL;
1278         if (!(flags & MSG_XPG4_2)) {
1279                 /*
1280                  * In order to be compatible with the libsocket/sockmod
1281                  * implementation we set EOR for all send* calls.
1282                  */
1283                 flags |= MSG_EOR;
1284         }
1285         return (sendit(sock, &lmsg, &auio, flags));
1286 }
1287 
1288 /*
1289  * Uses the MSG_XPG4_2 flag to determine if the caller is using
1290  * struct omsghdr or struct nmsghdr.
1291  */
1292 ssize_t
1293 sendmsg(int sock, struct nmsghdr *msg, int flags)
1294 {
1295         struct nmsghdr lmsg;
1296         STRUCT_DECL(nmsghdr, u_lmsg);
1297         struct uio auio;
1298         struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1299         ssize_t iovsize = 0;
1300         int iovcnt;
1301         ssize_t len, rval;
1302         int i;
1303         model_t model;
1304 
1305         dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags));
1306 
1307         model = get_udatamodel();
1308         STRUCT_INIT(u_lmsg, model);
1309 
1310         if (flags & MSG_XPG4_2) {
1311                 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1312                     STRUCT_SIZE(u_lmsg)))
1313                         return (set_errno(EFAULT));
1314         } else {
1315                 /*
1316                  * Assumes that nmsghdr and omsghdr are identically shaped
1317                  * except for the added msg_flags field.
1318                  */
1319                 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1320                     SIZEOF_STRUCT(omsghdr, model)))
1321                         return (set_errno(EFAULT));
1322                 /*
1323                  * In order to be compatible with the libsocket/sockmod
1324                  * implementation we set EOR for all send* calls.
1325                  */
1326                 flags |= MSG_EOR;
1327         }
1328 
1329         /*
1330          * Code below us will kmem_alloc memory and hang it
1331          * off msg_control and msg_name fields. This forces
1332          * us to copy the structure to its native form.
1333          */
1334         lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1335         lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1336         lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1337         lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1338         lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1339         lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1340         lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1341 
1342         iovcnt = lmsg.msg_iovlen;
1343 
1344         if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1345                 /*
1346                  * Unless this is XPG 4.2 we allow iovcnt == 0 to
1347                  * be compatible with SunOS 4.X and 4.4BSD.
1348                  */
1349                 if (iovcnt != 0 || (flags & MSG_XPG4_2))
1350                         return (set_errno(EMSGSIZE));
1351         }
1352 
1353         if (iovcnt > IOV_MAX_STACK) {
1354                 iovsize = iovcnt * sizeof (struct iovec);
1355                 aiov = kmem_alloc(iovsize, KM_SLEEP);
1356         }
1357 
1358 #ifdef _SYSCALL32_IMPL
1359         /*
1360          * 32-bit callers need to have their iovec expanded, while ensuring
1361          * that they can't move more than 2Gbytes of data in a single call.
1362          */
1363         if (model == DATAMODEL_ILP32) {
1364                 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1365                 ssize_t iov32size;
1366                 ssize32_t count32;
1367 
1368                 iov32size = iovcnt * sizeof (struct iovec32);
1369                 if (iovsize != 0)
1370                         aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1371 
1372                 if (iovcnt != 0 &&
1373                     copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1374                         if (iovsize != 0) {
1375                                 kmem_free(aiov32, iov32size);
1376                                 kmem_free(aiov, iovsize);
1377                         }
1378 
1379                         return (set_errno(EFAULT));
1380                 }
1381 
1382                 count32 = 0;
1383                 for (i = 0; i < iovcnt; i++) {
1384                         ssize32_t iovlen32;
1385 
1386                         iovlen32 = aiov32[i].iov_len;
1387                         count32 += iovlen32;
1388                         if (iovlen32 < 0 || count32 < 0) {
1389                                 if (iovsize != 0) {
1390                                         kmem_free(aiov32, iov32size);
1391                                         kmem_free(aiov, iovsize);
1392                                 }
1393 
1394                                 return (set_errno(EINVAL));
1395                         }
1396 
1397                         aiov[i].iov_len = iovlen32;
1398                         aiov[i].iov_base =
1399                             (caddr_t)(uintptr_t)aiov32[i].iov_base;
1400                 }
1401 
1402                 if (iovsize != 0)
1403                         kmem_free(aiov32, iov32size);
1404         } else
1405 #endif /* _SYSCALL32_IMPL */
1406         if (iovcnt != 0 &&
1407             copyin(lmsg.msg_iov, aiov,
1408             (unsigned)iovcnt * sizeof (struct iovec))) {
1409                 if (iovsize != 0)
1410                         kmem_free(aiov, iovsize);
1411 
1412                 return (set_errno(EFAULT));
1413         }
1414         len = 0;
1415         for (i = 0; i < iovcnt; i++) {
1416                 ssize_t iovlen = aiov[i].iov_len;
1417                 len += iovlen;
1418                 if (iovlen < 0 || len < 0) {
1419                         if (iovsize != 0)
1420                                 kmem_free(aiov, iovsize);
1421 
1422                         return (set_errno(EINVAL));
1423                 }
1424         }
1425         auio.uio_loffset = 0;
1426         auio.uio_iov = aiov;
1427         auio.uio_iovcnt = iovcnt;
1428         auio.uio_resid = len;
1429         auio.uio_segflg = UIO_USERSPACE;
1430         auio.uio_limit = 0;
1431 
1432         rval = sendit(sock, &lmsg, &auio, flags);
1433 
1434         if (iovsize != 0)
1435                 kmem_free(aiov, iovsize);
1436 
1437         return (rval);
1438 }
1439 
1440 ssize_t
1441 sendto(int sock, void *buffer, size_t len, int flags,
1442     struct sockaddr *name, socklen_t namelen)
1443 {
1444         struct nmsghdr lmsg;
1445         struct uio auio;
1446         struct iovec aiov[1];
1447 
1448         dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n",
1449             sock, buffer, len, flags, (void *)name, namelen));
1450 
1451         if ((ssize_t)len < 0) {
1452                 return (set_errno(EINVAL));
1453         }
1454 
1455         aiov[0].iov_base = buffer;
1456         aiov[0].iov_len = len;
1457         auio.uio_loffset = 0;
1458         auio.uio_iov = aiov;
1459         auio.uio_iovcnt = 1;
1460         auio.uio_resid = len;
1461         auio.uio_segflg = UIO_USERSPACE;
1462         auio.uio_limit = 0;
1463 
1464         lmsg.msg_name = (char *)name;
1465         lmsg.msg_namelen = namelen;
1466         lmsg.msg_control = NULL;
1467         if (!(flags & MSG_XPG4_2)) {
1468                 /*
1469                  * In order to be compatible with the libsocket/sockmod
1470                  * implementation we set EOR for all send* calls.
1471                  */
1472                 flags |= MSG_EOR;
1473         }
1474         return (sendit(sock, &lmsg, &auio, flags));
1475 }
1476 
1477 /*ARGSUSED3*/
1478 int
1479 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1480 {
1481         struct sonode *so;
1482         int error;
1483         socklen_t namelen;
1484         socklen_t sock_addrlen;
1485         struct sockaddr *sock_addrp;
1486 
1487         dprint(1, ("getpeername(%d, %p, %p)\n",
1488             sock, (void *)name, (void *)namelenp));
1489 
1490         if ((so = getsonode(sock, &error, NULL)) == NULL)
1491                 goto bad;
1492 
1493         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1494         if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1495             (name == NULL && namelen != 0)) {
1496                 error = EFAULT;
1497                 goto rel_out;
1498         }
1499         sock_addrlen = so->so_max_addr_len;
1500         sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1501 
1502         if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen,
1503             B_FALSE, CRED())) == 0) {
1504                 ASSERT(sock_addrlen <= so->so_max_addr_len);
1505                 error = copyout_name(name, namelen, namelenp,
1506                     (void *)sock_addrp, sock_addrlen);
1507         }
1508         kmem_free(sock_addrp, so->so_max_addr_len);
1509 rel_out:
1510         releasef(sock);
1511 bad:    return (error != 0 ? set_errno(error) : 0);
1512 }
1513 
1514 /*ARGSUSED3*/
1515 int
1516 getsockname(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1517 {
1518         struct sonode *so;
1519         int error;
1520         socklen_t namelen, sock_addrlen;
1521         struct sockaddr *sock_addrp;
1522 
1523         dprint(1, ("getsockname(%d, %p, %p)\n",
1524             sock, (void *)name, (void *)namelenp));
1525 
1526         if ((so = getsonode(sock, &error, NULL)) == NULL)
1527                 goto bad;
1528 
1529         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1530         if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1531             (name == NULL && namelen != 0)) {
1532                 error = EFAULT;
1533                 goto rel_out;
1534         }
1535 
1536         sock_addrlen = so->so_max_addr_len;
1537         sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1538         if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen,
1539             CRED())) == 0) {
1540                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1541                 ASSERT(sock_addrlen <= so->so_max_addr_len);
1542                 error = copyout_name(name, namelen, namelenp,
1543                     (void *)sock_addrp, sock_addrlen);
1544         }
1545         kmem_free(sock_addrp, so->so_max_addr_len);
1546 rel_out:
1547         releasef(sock);
1548 bad:    return (error != 0 ? set_errno(error) : 0);
1549 }
1550 
1551 /*ARGSUSED5*/
1552 int
1553 getsockopt(int sock, int level, int option_name, void *option_value,
1554     socklen_t *option_lenp, int version)
1555 {
1556         struct sonode *so;
1557         socklen_t optlen, optlen_res;
1558         void *optval;
1559         int error;
1560 
1561         dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n",
1562             sock, level, option_name, option_value, (void *)option_lenp));
1563 
1564         if ((so = getsonode(sock, &error, NULL)) == NULL)
1565                 return (set_errno(error));
1566 
1567         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1568         if (copyin(option_lenp, &optlen, sizeof (optlen))) {
1569                 releasef(sock);
1570                 return (set_errno(EFAULT));
1571         }
1572         /*
1573          * Verify that the length is not excessive to prevent
1574          * an application from consuming all of kernel memory.
1575          */
1576         if (optlen > SO_MAXARGSIZE) {
1577                 error = EINVAL;
1578                 releasef(sock);
1579                 return (set_errno(error));
1580         }
1581         optval = kmem_alloc(optlen, KM_SLEEP);
1582         optlen_res = optlen;
1583         error = socket_getsockopt(so, level, option_name, optval,
1584             &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2,
1585             CRED());
1586         releasef(sock);
1587         if (error) {
1588                 kmem_free(optval, optlen);
1589                 return (set_errno(error));
1590         }
1591         error = copyout_arg(option_value, optlen, option_lenp,
1592             optval, optlen_res);
1593         kmem_free(optval, optlen);
1594         if (error)
1595                 return (set_errno(error));
1596         return (0);
1597 }
1598 
1599 /*ARGSUSED5*/
1600 int
1601 setsockopt(int sock, int level, int option_name, void *option_value,
1602     socklen_t option_len, int version)
1603 {
1604         struct sonode *so;
1605         intptr_t buffer[2];
1606         void *optval = NULL;
1607         int error;
1608 
1609         dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n",
1610             sock, level, option_name, option_value, option_len));
1611 
1612         if ((so = getsonode(sock, &error, NULL)) == NULL)
1613                 return (set_errno(error));
1614 
1615         if (option_value != NULL) {
1616                 if (option_len != 0) {
1617                         /*
1618                          * Verify that the length is not excessive to prevent
1619                          * an application from consuming all of kernel memory.
1620                          */
1621                         if (option_len > SO_MAXARGSIZE) {
1622                                 error = EINVAL;
1623                                 goto done2;
1624                         }
1625                         optval = option_len <= sizeof (buffer) ?
1626                             &buffer : kmem_alloc((size_t)option_len, KM_SLEEP);
1627                         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1628                         if (copyin(option_value, optval, (size_t)option_len)) {
1629                                 error = EFAULT;
1630                                 goto done1;
1631                         }
1632                 }
1633         } else
1634                 option_len = 0;
1635 
1636         error = socket_setsockopt(so, level, option_name, optval,
1637             (t_uscalar_t)option_len, CRED());
1638 done1:
1639         if (optval != buffer)
1640                 kmem_free(optval, (size_t)option_len);
1641 done2:
1642         releasef(sock);
1643         if (error)
1644                 return (set_errno(error));
1645         return (0);
1646 }
1647 
1648 static int
1649 sockconf_add_sock(int family, int type, int protocol, char *name)
1650 {
1651         int error = 0;
1652         char *kdevpath = NULL;
1653         char *kmodule = NULL;
1654         char *buf = NULL;
1655         size_t pathlen = 0;
1656         struct sockparams *sp;
1657 
1658         if (name == NULL)
1659                 return (EINVAL);
1660         /*
1661          * Copyin the name.
1662          * This also makes it possible to check for too long pathnames.
1663          * Compress the space needed for the name before passing it
1664          * to soconfig - soconfig will store the string until
1665          * the configuration is removed.
1666          */
1667         buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1668         if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) {
1669                 kmem_free(buf, MAXPATHLEN);
1670                 return (error);
1671         }
1672         if (strncmp(buf, "/dev", strlen("/dev")) == 0) {
1673                 /* For device */
1674 
1675                 /*
1676                  * Special handling for NCA:
1677                  *
1678                  * DEV_NCA is never opened even if an application
1679                  * requests for AF_NCA. The device opened is instead a
1680                  * predefined AF_INET transport (NCA_INET_DEV).
1681                  *
1682                  * Prior to Volo (PSARC/2007/587) NCA would determine
1683                  * the device using a lookup, which worked then because
1684                  * all protocols were based on TPI. Since TPI is no
1685                  * longer the default, we have to explicitly state
1686                  * which device to use.
1687                  */
1688                 if (strcmp(buf, NCA_DEV) == 0) {
1689                         /* only support entry <28, 2, 0> */
1690                         if (family != AF_NCA || type != SOCK_STREAM ||
1691                             protocol != 0) {
1692                                 kmem_free(buf, MAXPATHLEN);
1693                                 return (EINVAL);
1694                         }
1695 
1696                         pathlen = strlen(NCA_INET_DEV) + 1;
1697                         kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1698                         bcopy(NCA_INET_DEV, kdevpath, pathlen);
1699                         kdevpath[pathlen - 1] = '\0';
1700                 } else {
1701                         kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1702                         bcopy(buf, kdevpath, pathlen);
1703                         kdevpath[pathlen - 1] = '\0';
1704                 }
1705         } else {
1706                 /* For socket module */
1707                 kmodule = kmem_alloc(pathlen, KM_SLEEP);
1708                 bcopy(buf, kmodule, pathlen);
1709                 kmodule[pathlen - 1] = '\0';
1710                 pathlen = 0;
1711         }
1712         kmem_free(buf, MAXPATHLEN);
1713 
1714         /* sockparams_create frees mod name and devpath upon failure */
1715         sp = sockparams_create(family, type, protocol, kmodule,
1716             kdevpath, pathlen, 0, KM_SLEEP, &error);
1717         if (sp != NULL) {
1718                 error = sockparams_add(sp);
1719                 if (error != 0)
1720                         sockparams_destroy(sp);
1721         }
1722 
1723         return (error);
1724 }
1725 
1726 static int
1727 sockconf_remove_sock(int family, int type, int protocol)
1728 {
1729         return (sockparams_delete(family, type, protocol));
1730 }
1731 
1732 static int
1733 sockconfig_remove_filter(const char *uname)
1734 {
1735         char kname[SOF_MAXNAMELEN];
1736         size_t len;
1737         int error;
1738         sof_entry_t *ent;
1739 
1740         if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0)
1741                 return (error);
1742 
1743         ent = sof_entry_remove_by_name(kname);
1744         if (ent == NULL)
1745                 return (ENXIO);
1746 
1747         mutex_enter(&ent->sofe_lock);
1748         ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED));
1749         if (ent->sofe_refcnt == 0) {
1750                 mutex_exit(&ent->sofe_lock);
1751                 sof_entry_free(ent);
1752         } else {
1753                 /* let the last socket free the filter */
1754                 ent->sofe_flags |= SOFEF_CONDEMED;
1755                 mutex_exit(&ent->sofe_lock);
1756         }
1757 
1758         return (0);
1759 }
1760 
1761 static int
1762 sockconfig_add_filter(const char *uname, void *ufilpropp)
1763 {
1764         struct sockconfig_filter_props filprop;
1765         sof_entry_t *ent;
1766         int error;
1767         size_t tuplesz, len;
1768         char hintbuf[SOF_MAXNAMELEN];
1769 
1770         ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP);
1771         mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL);
1772 
1773         if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN,
1774             &len)) != 0) {
1775                 sof_entry_free(ent);
1776                 return (error);
1777         }
1778 
1779         if (get_udatamodel() == DATAMODEL_NATIVE) {
1780                 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) {
1781                         sof_entry_free(ent);
1782                         return (EFAULT);
1783                 }
1784         }
1785 #ifdef  _SYSCALL32_IMPL
1786         else {
1787                 struct sockconfig_filter_props32 filprop32;
1788 
1789                 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) {
1790                         sof_entry_free(ent);
1791                         return (EFAULT);
1792                 }
1793                 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname;
1794                 filprop.sfp_autoattach = filprop32.sfp_autoattach;
1795                 filprop.sfp_hint = filprop32.sfp_hint;
1796                 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg;
1797                 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt;
1798                 filprop.sfp_socktuple =
1799                     (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple;
1800         }
1801 #endif  /* _SYSCALL32_IMPL */
1802 
1803         if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname,
1804             sizeof (ent->sofe_modname), &len)) != 0) {
1805                 sof_entry_free(ent);
1806                 return (error);
1807         }
1808 
1809         /*
1810          * A filter must specify at least one socket tuple.
1811          */
1812         if (filprop.sfp_socktuple_cnt == 0 ||
1813             filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) {
1814                 sof_entry_free(ent);
1815                 return (EINVAL);
1816         }
1817         ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG;
1818         ent->sofe_hint = filprop.sfp_hint;
1819 
1820         /*
1821          * Verify the hint, and copy in the hint argument, if necessary.
1822          */
1823         switch (ent->sofe_hint) {
1824         case SOF_HINT_BEFORE:
1825         case SOF_HINT_AFTER:
1826                 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf,
1827                     sizeof (hintbuf), &len)) != 0) {
1828                         sof_entry_free(ent);
1829                         return (error);
1830                 }
1831                 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP);
1832                 bcopy(hintbuf, ent->sofe_hintarg, len);
1833                 /* FALLTHRU */
1834         case SOF_HINT_TOP:
1835         case SOF_HINT_BOTTOM:
1836                 /* hints cannot be used with programmatic filters */
1837                 if (ent->sofe_flags & SOFEF_PROG) {
1838                         sof_entry_free(ent);
1839                         return (EINVAL);
1840                 }
1841                 break;
1842         case SOF_HINT_NONE:
1843                 break;
1844         default:
1845                 /* bad hint value */
1846                 sof_entry_free(ent);
1847                 return (EINVAL);
1848         }
1849 
1850         ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt;
1851         tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt;
1852         ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP);
1853 
1854         if (get_udatamodel() == DATAMODEL_NATIVE) {
1855                 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple,
1856                     tuplesz)) {
1857                         sof_entry_free(ent);
1858                         return (EFAULT);
1859                 }
1860         }
1861 #ifdef  _SYSCALL32_IMPL
1862         else {
1863                 int i;
1864                 caddr_t data = (caddr_t)filprop.sfp_socktuple;
1865                 sof_socktuple_t *tup = ent->sofe_socktuple;
1866                 sof_socktuple32_t tup32;
1867 
1868                 tup = ent->sofe_socktuple;
1869                 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) {
1870                         ASSERT(tup < ent->sofe_socktuple + tuplesz);
1871 
1872                         if (copyin(data, &tup32, sizeof (tup32)) != 0) {
1873                                 sof_entry_free(ent);
1874                                 return (EFAULT);
1875                         }
1876                         tup->sofst_family = tup32.sofst_family;
1877                         tup->sofst_type = tup32.sofst_type;
1878                         tup->sofst_protocol = tup32.sofst_protocol;
1879 
1880                         data += sizeof (tup32);
1881                 }
1882         }
1883 #endif  /* _SYSCALL32_IMPL */
1884 
1885         /* Sockets can start using the filter as soon as the filter is added */
1886         if ((error = sof_entry_add(ent)) != 0)
1887                 sof_entry_free(ent);
1888 
1889         return (error);
1890 }
1891 
1892 /*
1893  * Socket configuration system call. It is used to add and remove
1894  * socket types.
1895  */
1896 int
1897 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4)
1898 {
1899         int error = 0;
1900 
1901         if (secpolicy_net_config(CRED(), B_FALSE) != 0)
1902                 return (set_errno(EPERM));
1903 
1904         if (sockfs_defer_nl7c_init) {
1905                 nl7c_init();
1906                 sockfs_defer_nl7c_init = 0;
1907         }
1908 
1909         switch (cmd) {
1910         case SOCKCONFIG_ADD_SOCK:
1911                 error = sockconf_add_sock((int)(uintptr_t)arg1,
1912                     (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4);
1913                 break;
1914         case SOCKCONFIG_REMOVE_SOCK:
1915                 error = sockconf_remove_sock((int)(uintptr_t)arg1,
1916                     (int)(uintptr_t)arg2, (int)(uintptr_t)arg3);
1917                 break;
1918         case SOCKCONFIG_ADD_FILTER:
1919                 error = sockconfig_add_filter((const char *)arg1, arg2);
1920                 break;
1921         case SOCKCONFIG_REMOVE_FILTER:
1922                 error = sockconfig_remove_filter((const char *)arg1);
1923                 break;
1924         case SOCKCONFIG_GET_SOCKTABLE:
1925                 error = sockparams_copyout_socktable((int)(uintptr_t)arg1);
1926                 break;
1927         default:
1928 #ifdef  DEBUG
1929                 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd);
1930 #endif
1931                 error = EINVAL;
1932                 break;
1933         }
1934 
1935         if (error != 0) {
1936                 eprintline(error);
1937                 return (set_errno(error));
1938         }
1939         return (0);
1940 }
1941 
1942 
1943 /*
1944  * Sendfile is implemented through two schemes, direct I/O or by
1945  * caching in the filesystem page cache. We cache the input file by
1946  * default and use direct I/O only if sendfile_max_size is set
1947  * appropriately as explained below. Note that this logic is consistent
1948  * with other filesystems where caching is turned on by default
1949  * unless explicitly turned off by using the DIRECTIO ioctl.
1950  *
1951  * We choose a slightly different scheme here. One can turn off
1952  * caching by setting sendfile_max_size to 0. One can also enable
1953  * caching of files <= sendfile_max_size by setting sendfile_max_size
1954  * to an appropriate value. By default sendfile_max_size is set to the
1955  * maximum value so that all files are cached. In future, we may provide
1956  * better interfaces for caching the file.
1957  *
1958  * Sendfile through Direct I/O (Zero copy)
1959  * --------------------------------------
1960  *
1961  * As disks are normally slower than the network, we can't have a
1962  * single thread that reads the disk and writes to the network. We
1963  * need to have parallelism. This is done by having the sendfile
1964  * thread create another thread that reads from the filesystem
1965  * and queues it for network processing. In this scheme, the data
1966  * is never copied anywhere i.e it is zero copy unlike the other
1967  * scheme.
1968  *
1969  * We have a sendfile queue (snfq) where each sendfile
1970  * request (snf_req_t) is queued for processing by a thread. Number
1971  * of threads is dynamically allocated and they exit if they are idling
1972  * beyond a specified amount of time. When each request (snf_req_t) is
1973  * processed by a thread, it produces a number of mblk_t structures to
1974  * be consumed by the sendfile thread. snf_deque and snf_enque are
1975  * used for consuming and producing mblks. Size of the filesystem
1976  * read is determined by the tunable (sendfile_read_size). A single
1977  * mblk holds sendfile_read_size worth of data (except the last
1978  * read of the file) which is sent down as a whole to the network.
1979  * sendfile_read_size is set to 1 MB as this seems to be the optimal
1980  * value for the UFS filesystem backed by a striped storage array.
1981  *
1982  * Synchronisation between read (producer) and write (consumer) threads.
1983  * --------------------------------------------------------------------
1984  *
1985  * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while
1986  * adding and deleting items in this list. Error can happen anytime
1987  * during read or write. There could be unprocessed mblks in the
1988  * sr_ib_XXX list when a read or write error occurs. Whenever error
1989  * is encountered, we need two things to happen :
1990  *
1991  * a) One of the threads need to clean the mblks.
1992  * b) When one thread encounters an error, the other should stop.
1993  *
1994  * For (a), we don't want to penalize the reader thread as it could do
1995  * some useful work processing other requests. For (b), the error can
1996  * be detected by examining sr_read_error or sr_write_error.
1997  * sr_lock protects sr_read_error and sr_write_error. If both reader and
1998  * writer encounters error, we need to report the write error back to
1999  * the application as that's what would have happened if the operations
2000  * were done sequentially. With this in mind, following should work :
2001  *
2002  *      - Check for errors before read or write.
2003  *      - If the reader encounters error, set the error in sr_read_error.
2004  *        Check sr_write_error, if it is set, send cv_signal as it is
2005  *        waiting for reader to complete. If it is not set, the writer
2006  *        is either running sinking data to the network or blocked
2007  *        because of flow control. For handling the latter case, we
2008  *        always send a signal. In any case, it will examine sr_read_error
2009  *        and return. sr_read_error is marked with SR_READ_DONE to tell
2010  *        the writer that the reader is done in all the cases.
2011  *      - If the writer encounters error, set the error in sr_write_error.
2012  *        The reader thread is either blocked because of flow control or
2013  *        running reading data from the disk. For the former, we need to
2014  *        wakeup the thread. Again to keep it simple, we always wake up
2015  *        the reader thread. Then, wait for the read thread to complete
2016  *        if it is not done yet. Cleanup and return.
2017  *
2018  * High and low water marks for the read thread.
2019  * --------------------------------------------
2020  *
2021  * If sendfile() is used to send data over a slow network, we need to
2022  * make sure that the read thread does not produce data at a faster
2023  * rate than the network. This can happen if the disk is faster than
2024  * the network. In such a case, we don't want to build a very large queue.
2025  * But we would still like to get all of the network throughput possible.
2026  * This implies that network should never block waiting for data.
2027  * As there are lot of disk throughput/network throughput combinations
2028  * possible, it is difficult to come up with an accurate number.
2029  * A typical 10K RPM disk has a max seek latency 17ms and rotational
2030  * latency of 3ms for reading a disk block. Thus, the total latency to
2031  * initiate a new read, transfer data from the disk and queue for
2032  * transmission would take about a max of 25ms. Todays max transfer rate
2033  * for network is 100MB/sec. If the thread is blocked because of flow
2034  * control, it would take 25ms to get new data ready for transmission.
2035  * We have to make sure that network is not idling, while we are initiating
2036  * new transfers. So, at 100MB/sec, to keep network busy we would need
2037  * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data.
2038  * We need to pick a high water mark so that the woken up thread would
2039  * do considerable work before blocking again to prevent thrashing. Currently,
2040  * we pick this to be 10 times that of the low water mark.
2041  *
2042  * Sendfile with segmap caching (One copy from page cache to mblks).
2043  * ----------------------------------------------------------------
2044  *
2045  * We use the segmap cache for caching the file, if the size of file
2046  * is <= sendfile_max_size. In this case we don't use threads as VM
2047  * is reasonably fast enough to keep up with the network. If the underlying
2048  * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth
2049  * of data into segmap space, and use the virtual address from segmap
2050  * directly through desballoc() to avoid copy. Once the transport is done
2051  * with the data, the mapping will be released through segmap_release()
2052  * called by the call-back routine.
2053  *
2054  * If zero-copy is not allowed by the transport, we simply call VOP_READ()
2055  * to copy the data from the filesystem into our temporary network buffer.
2056  *
2057  * To disable caching, set sendfile_max_size to 0.
2058  */
2059 
2060 uint_t sendfile_read_size = 1024 * 1024;
2061 #define SENDFILE_REQ_LOWAT      3 * 1024 * 1024
2062 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT;
2063 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT;
2064 struct sendfile_stats sf_stats;
2065 struct sendfile_queue *snfq;
2066 clock_t snfq_timeout;
2067 off64_t sendfile_max_size;
2068 
2069 static void snf_enque(snf_req_t *, mblk_t *);
2070 static mblk_t *snf_deque(snf_req_t *);
2071 
2072 void
2073 sendfile_init(void)
2074 {
2075         snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP);
2076 
2077         mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL);
2078         cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL);
2079         snfq->snfq_max_threads = max_ncpus;
2080         snfq_timeout = SNFQ_TIMEOUT;
2081         /* Cache all files by default. */
2082         sendfile_max_size = MAXOFFSET_T;
2083 }
2084 
2085 /*
2086  * Queues a mblk_t for network processing.
2087  */
2088 static void
2089 snf_enque(snf_req_t *sr, mblk_t *mp)
2090 {
2091         mp->b_next = NULL;
2092         mutex_enter(&sr->sr_lock);
2093         if (sr->sr_mp_head == NULL) {
2094                 sr->sr_mp_head = sr->sr_mp_tail = mp;
2095                 cv_signal(&sr->sr_cv);
2096         } else {
2097                 sr->sr_mp_tail->b_next = mp;
2098                 sr->sr_mp_tail = mp;
2099         }
2100         sr->sr_qlen += MBLKL(mp);
2101         while ((sr->sr_qlen > sr->sr_hiwat) &&
2102             (sr->sr_write_error == 0)) {
2103                 sf_stats.ss_full_waits++;
2104                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2105         }
2106         mutex_exit(&sr->sr_lock);
2107 }
2108 
2109 /*
2110  * De-queues a mblk_t for network processing.
2111  */
2112 static mblk_t *
2113 snf_deque(snf_req_t *sr)
2114 {
2115         mblk_t *mp;
2116 
2117         mutex_enter(&sr->sr_lock);
2118         /*
2119          * If we have encountered an error on read or read is
2120          * completed and no more mblks, return NULL.
2121          * We need to check for NULL sr_mp_head also as
2122          * the reads could have completed and there is
2123          * nothing more to come.
2124          */
2125         if (((sr->sr_read_error & ~SR_READ_DONE) != 0) ||
2126             ((sr->sr_read_error & SR_READ_DONE) &&
2127             sr->sr_mp_head == NULL)) {
2128                 mutex_exit(&sr->sr_lock);
2129                 return (NULL);
2130         }
2131         /*
2132          * To start with neither SR_READ_DONE is marked nor
2133          * the error is set. When we wake up from cv_wait,
2134          * following are the possibilities :
2135          *
2136          *      a) sr_read_error is zero and mblks are queued.
2137          *      b) sr_read_error is set to SR_READ_DONE
2138          *         and mblks are queued.
2139          *      c) sr_read_error is set to SR_READ_DONE
2140          *         and no mblks.
2141          *      d) sr_read_error is set to some error other
2142          *         than SR_READ_DONE.
2143          */
2144 
2145         while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) {
2146                 sf_stats.ss_empty_waits++;
2147                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2148         }
2149         /* Handle (a) and (b) first  - the normal case. */
2150         if (((sr->sr_read_error & ~SR_READ_DONE) == 0) &&
2151             (sr->sr_mp_head != NULL)) {
2152                 mp = sr->sr_mp_head;
2153                 sr->sr_mp_head = mp->b_next;
2154                 sr->sr_qlen -= MBLKL(mp);
2155                 if (sr->sr_qlen < sr->sr_lowat)
2156                         cv_signal(&sr->sr_cv);
2157                 mutex_exit(&sr->sr_lock);
2158                 mp->b_next = NULL;
2159                 return (mp);
2160         }
2161         /* Handle (c) and (d). */
2162         mutex_exit(&sr->sr_lock);
2163         return (NULL);
2164 }
2165 
2166 /*
2167  * Reads data from the filesystem and queues it for network processing.
2168  */
2169 void
2170 snf_async_read(snf_req_t *sr)
2171 {
2172         size_t iosize;
2173         u_offset_t fileoff;
2174         u_offset_t size;
2175         int ret_size;
2176         int error;
2177         file_t *fp;
2178         mblk_t *mp;
2179         struct vnode *vp;
2180         int extra = 0;
2181         int maxblk = 0;
2182         int wroff = 0;
2183         struct sonode *so;
2184 
2185         fp = sr->sr_fp;
2186         size = sr->sr_file_size;
2187         fileoff = sr->sr_file_off;
2188 
2189         /*
2190          * Ignore the error for filesystems that doesn't support DIRECTIO.
2191          */
2192         (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0,
2193             kcred, NULL, NULL);
2194 
2195         vp = sr->sr_vp;
2196         if (vp->v_type == VSOCK) {
2197                 stdata_t *stp;
2198 
2199                 /*
2200                  * Get the extra space to insert a header and a trailer.
2201                  */
2202                 so = VTOSO(vp);
2203                 stp = vp->v_stream;
2204                 if (stp == NULL) {
2205                         wroff = so->so_proto_props.sopp_wroff;
2206                         maxblk = so->so_proto_props.sopp_maxblk;
2207                         extra = wroff + so->so_proto_props.sopp_tail;
2208                 } else {
2209                         wroff = (int)(stp->sd_wroff);
2210                         maxblk = (int)(stp->sd_maxblk);
2211                         extra = wroff + (int)(stp->sd_tail);
2212                 }
2213         }
2214 
2215         while ((size != 0) && (sr->sr_write_error == 0)) {
2216 
2217                 iosize = (int)MIN(sr->sr_maxpsz, size);
2218 
2219                 /*
2220                  * Socket filters can limit the mblk size,
2221                  * so limit reads to maxblk if there are
2222                  * filters present.
2223                  */
2224                 if (vp->v_type == VSOCK &&
2225                     so->so_filter_active > 0 && maxblk != INFPSZ)
2226                         iosize = (int)MIN(iosize, maxblk);
2227 
2228                 if (is_system_labeled()) {
2229                         mp = allocb_cred(iosize + extra, CRED(),
2230                             curproc->p_pid);
2231                 } else {
2232                         mp = allocb(iosize + extra, BPRI_MED);
2233                 }
2234                 if (mp == NULL) {
2235                         error = EAGAIN;
2236                         break;
2237                 }
2238 
2239                 mp->b_rptr += wroff;
2240 
2241                 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize);
2242 
2243                 /* Error or Reached EOF ? */
2244                 if ((error != 0) || (ret_size == 0)) {
2245                         freeb(mp);
2246                         break;
2247                 }
2248                 mp->b_wptr = mp->b_rptr + ret_size;
2249 
2250                 snf_enque(sr, mp);
2251                 size -= ret_size;
2252                 fileoff += ret_size;
2253         }
2254         (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0,
2255             kcred, NULL, NULL);
2256         mutex_enter(&sr->sr_lock);
2257         sr->sr_read_error = error;
2258         sr->sr_read_error |= SR_READ_DONE;
2259         cv_signal(&sr->sr_cv);
2260         mutex_exit(&sr->sr_lock);
2261 }
2262 
2263 void
2264 snf_async_thread(void)
2265 {
2266         snf_req_t *sr;
2267         callb_cpr_t cprinfo;
2268         clock_t time_left = 1;
2269 
2270         CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq");
2271 
2272         mutex_enter(&snfq->snfq_lock);
2273         for (;;) {
2274                 /*
2275                  * If we didn't find a entry, then block until woken up
2276                  * again and then look through the queues again.
2277                  */
2278                 while ((sr = snfq->snfq_req_head) == NULL) {
2279                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
2280                         if (time_left <= 0) {
2281                                 snfq->snfq_svc_threads--;
2282                                 CALLB_CPR_EXIT(&cprinfo);
2283                                 thread_exit();
2284                                 /* NOTREACHED */
2285                         }
2286                         snfq->snfq_idle_cnt++;
2287 
2288                         time_left = cv_reltimedwait(&snfq->snfq_cv,
2289                             &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK);
2290                         snfq->snfq_idle_cnt--;
2291 
2292                         CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock);
2293                 }
2294                 snfq->snfq_req_head = sr->sr_next;
2295                 snfq->snfq_req_cnt--;
2296                 mutex_exit(&snfq->snfq_lock);
2297                 snf_async_read(sr);
2298                 mutex_enter(&snfq->snfq_lock);
2299         }
2300 }
2301 
2302 
2303 snf_req_t *
2304 create_thread(int operation, struct vnode *vp, file_t *fp,
2305     u_offset_t fileoff, u_offset_t size)
2306 {
2307         snf_req_t *sr;
2308         stdata_t *stp;
2309 
2310         sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP);
2311 
2312         sr->sr_vp = vp;
2313         sr->sr_fp = fp;
2314         stp = vp->v_stream;
2315 
2316         /*
2317          * store sd_qn_maxpsz into sr_maxpsz while we have stream head.
2318          * stream might be closed before thread returns from snf_async_read.
2319          */
2320         if (stp != NULL && stp->sd_qn_maxpsz > 0) {
2321                 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz);
2322         } else {
2323                 sr->sr_maxpsz = MAXBSIZE;
2324         }
2325 
2326         sr->sr_operation = operation;
2327         sr->sr_file_off = fileoff;
2328         sr->sr_file_size = size;
2329         sr->sr_hiwat = sendfile_req_hiwat;
2330         sr->sr_lowat = sendfile_req_lowat;
2331         mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL);
2332         cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL);
2333         /*
2334          * See whether we need another thread for servicing this
2335          * request. If there are already enough requests queued
2336          * for the threads, create one if not exceeding
2337          * snfq_max_threads.
2338          */
2339         mutex_enter(&snfq->snfq_lock);
2340         if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt &&
2341             snfq->snfq_svc_threads < snfq->snfq_max_threads) {
2342                 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0,
2343                     TS_RUN, minclsyspri);
2344                 snfq->snfq_svc_threads++;
2345         }
2346         if (snfq->snfq_req_head == NULL) {
2347                 snfq->snfq_req_head = snfq->snfq_req_tail = sr;
2348                 cv_signal(&snfq->snfq_cv);
2349         } else {
2350                 snfq->snfq_req_tail->sr_next = sr;
2351                 snfq->snfq_req_tail = sr;
2352         }
2353         snfq->snfq_req_cnt++;
2354         mutex_exit(&snfq->snfq_lock);
2355         return (sr);
2356 }
2357 
2358 int
2359 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size,
2360     ssize_t *count)
2361 {
2362         snf_req_t *sr;
2363         mblk_t *mp;
2364         int iosize;
2365         int error = 0;
2366         short fflag;
2367         struct vnode *vp;
2368         int ksize;
2369         struct nmsghdr msg;
2370 
2371         ksize = 0;
2372         *count = 0;
2373         bzero(&msg, sizeof (msg));
2374 
2375         vp = fp->f_vnode;
2376         fflag = fp->f_flag;
2377         if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL)
2378                 return (EAGAIN);
2379 
2380         /*
2381          * We check for read error in snf_deque. It has to check
2382          * for successful READ_DONE and return NULL, and we might
2383          * as well make an additional check there.
2384          */
2385         while ((mp = snf_deque(sr)) != NULL) {
2386 
2387                 if (ISSIG(curthread, JUSTLOOKING)) {
2388                         freeb(mp);
2389                         error = EINTR;
2390                         break;
2391                 }
2392                 iosize = MBLKL(mp);
2393 
2394                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2395 
2396                 if (error != 0) {
2397                         if (mp != NULL)
2398                                 freeb(mp);
2399                         break;
2400                 }
2401                 ksize += iosize;
2402         }
2403         *count = ksize;
2404 
2405         mutex_enter(&sr->sr_lock);
2406         sr->sr_write_error = error;
2407         /* Look at the big comments on why we cv_signal here. */
2408         cv_signal(&sr->sr_cv);
2409 
2410         /* Wait for the reader to complete always. */
2411         while (!(sr->sr_read_error & SR_READ_DONE)) {
2412                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2413         }
2414         /* If there is no write error, check for read error. */
2415         if (error == 0)
2416                 error = (sr->sr_read_error & ~SR_READ_DONE);
2417 
2418         if (error != 0) {
2419                 mblk_t *next_mp;
2420 
2421                 mp = sr->sr_mp_head;
2422                 while (mp != NULL) {
2423                         next_mp = mp->b_next;
2424                         mp->b_next = NULL;
2425                         freeb(mp);
2426                         mp = next_mp;
2427                 }
2428         }
2429         mutex_exit(&sr->sr_lock);
2430         kmem_free(sr, sizeof (snf_req_t));
2431         return (error);
2432 }
2433 
2434 /* Maximum no.of pages allocated by vpm for sendfile at a time */
2435 #define SNF_VPMMAXPGS   (VPMMAXPGS/2)
2436 
2437 /*
2438  * Maximum no.of elements in the list returned by vpm, including
2439  * NULL for the last entry
2440  */
2441 #define SNF_MAXVMAPS    (SNF_VPMMAXPGS + 1)
2442 
2443 typedef struct {
2444         unsigned int    snfv_ref;
2445         frtn_t          snfv_frtn;
2446         vnode_t         *snfv_vp;
2447         struct vmap     snfv_vml[SNF_MAXVMAPS];
2448 } snf_vmap_desbinfo;
2449 
2450 typedef struct {
2451         frtn_t          snfi_frtn;
2452         caddr_t         snfi_base;
2453         uint_t          snfi_mapoff;
2454         size_t          snfi_len;
2455         vnode_t         *snfi_vp;
2456 } snf_smap_desbinfo;
2457 
2458 /*
2459  * The callback function used for vpm mapped mblks called when the last ref of
2460  * the mblk is dropped which normally occurs when TCP receives the ack. But it
2461  * can be the driver too due to lazy reclaim.
2462  */
2463 void
2464 snf_vmap_desbfree(snf_vmap_desbinfo *snfv)
2465 {
2466         ASSERT(snfv->snfv_ref != 0);
2467         if (atomic_dec_32_nv(&snfv->snfv_ref) == 0) {
2468                 vpm_unmap_pages(snfv->snfv_vml, S_READ);
2469                 VN_RELE(snfv->snfv_vp);
2470                 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2471         }
2472 }
2473 
2474 /*
2475  * The callback function used for segmap'ped mblks called when the last ref of
2476  * the mblk is dropped which normally occurs when TCP receives the ack. But it
2477  * can be the driver too due to lazy reclaim.
2478  */
2479 void
2480 snf_smap_desbfree(snf_smap_desbinfo *snfi)
2481 {
2482         if (! IS_KPM_ADDR(snfi->snfi_base)) {
2483                 /*
2484                  * We don't need to call segmap_fault(F_SOFTUNLOCK) for
2485                  * segmap_kpm as long as the latter never falls back to
2486                  * "use_segmap_range". (See segmap_getmapflt().)
2487                  *
2488                  * Using S_OTHER saves an redundant hat_setref() in
2489                  * segmap_unlock()
2490                  */
2491                 (void) segmap_fault(kas.a_hat, segkmap,
2492                     (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base +
2493                     snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len,
2494                     F_SOFTUNLOCK, S_OTHER);
2495         }
2496         (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED);
2497         VN_RELE(snfi->snfi_vp);
2498         kmem_free(snfi, sizeof (*snfi));
2499 }
2500 
2501 /*
2502  * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk.
2503  * When segmap is used, the mblk contains a segmap slot of no more
2504  * than MAXBSIZE.
2505  *
2506  * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained
2507  * in each iteration and sent by socket_sendmblk until an error occurs or
2508  * the requested size has been transferred. An mblk is esballoca'ed from
2509  * each mapped page and a chain of these mblk is sent to the transport layer.
2510  * vpm will be called to unmap the pages when all mblks have been freed by
2511  * free_func.
2512  *
2513  * At the end of the whole sendfile() operation, we wait till the data from
2514  * the last mblk is ack'ed by the transport before returning so that the
2515  * caller of sendfile() can safely modify the file content.
2516  *
2517  * The caller of this function should make sure that total_size does not exceed
2518  * the actual file size of fvp.
2519  */
2520 int
2521 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size,
2522     ssize_t *count, boolean_t nowait)
2523 {
2524         caddr_t base;
2525         int mapoff;
2526         vnode_t *vp;
2527         mblk_t *mp = NULL;
2528         int chain_size;
2529         int error;
2530         clock_t deadlk_wait;
2531         short fflag;
2532         int ksize;
2533         struct vattr va;
2534         boolean_t dowait = B_FALSE;
2535         struct nmsghdr msg;
2536 
2537         vp = fp->f_vnode;
2538         fflag = fp->f_flag;
2539         ksize = 0;
2540         bzero(&msg, sizeof (msg));
2541 
2542         for (;;) {
2543                 if (ISSIG(curthread, JUSTLOOKING)) {
2544                         error = EINTR;
2545                         break;
2546                 }
2547 
2548                 if (vpm_enable) {
2549                         snf_vmap_desbinfo *snfv;
2550                         mblk_t *nmp;
2551                         int mblk_size;
2552                         int maxsize;
2553                         int i;
2554 
2555                         mapoff = fileoff & PAGEOFFSET;
2556                         maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size);
2557 
2558                         snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo),
2559                             KM_SLEEP);
2560 
2561                         /*
2562                          * Get vpm mappings for maxsize with read access.
2563                          * If the pages aren't available yet, we get
2564                          * DEADLK, so wait and try again a little later using
2565                          * an increasing wait. We might be here a long time.
2566                          *
2567                          * If delay_sig returns EINTR, be sure to exit and
2568                          * pass it up to the caller.
2569                          */
2570                         deadlk_wait = 0;
2571                         while ((error = vpm_map_pages(fvp, fileoff,
2572                             (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml,
2573                             SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) {
2574                                 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2575                                 if ((error = delay_sig(deadlk_wait)) != 0) {
2576                                         break;
2577                                 }
2578                         }
2579                         if (error != 0) {
2580                                 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2581                                 error = (error == EINTR) ? EINTR : EIO;
2582                                 goto out;
2583                         }
2584                         snfv->snfv_frtn.free_func = snf_vmap_desbfree;
2585                         snfv->snfv_frtn.free_arg = (caddr_t)snfv;
2586 
2587                         /* Construct the mblk chain from the page mappings */
2588                         chain_size = 0;
2589                         for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) &&
2590                             total_size > 0; i++) {
2591                                 ASSERT(chain_size < maxsize);
2592                                 mblk_size = MIN(snfv->snfv_vml[i].vs_len -
2593                                     mapoff, total_size);
2594                                 nmp = esballoca(
2595                                     (uchar_t *)snfv->snfv_vml[i].vs_addr +
2596                                     mapoff, mblk_size, BPRI_HI,
2597                                     &snfv->snfv_frtn);
2598 
2599                                 /*
2600                                  * We return EAGAIN after unmapping the pages
2601                                  * if we cannot allocate the the head of the
2602                                  * chain. Otherwise, we continue sending the
2603                                  * mblks constructed so far.
2604                                  */
2605                                 if (nmp == NULL) {
2606                                         if (i == 0) {
2607                                                 vpm_unmap_pages(snfv->snfv_vml,
2608                                                     S_READ);
2609                                                 kmem_free(snfv,
2610                                                     sizeof (snf_vmap_desbinfo));
2611                                                 error = EAGAIN;
2612                                                 goto out;
2613                                         }
2614                                         break;
2615                                 }
2616                                 /* Mark this dblk with the zero-copy flag */
2617                                 nmp->b_datap->db_struioflag |= STRUIO_ZC;
2618                                 nmp->b_wptr += mblk_size;
2619                                 chain_size += mblk_size;
2620                                 fileoff += mblk_size;
2621                                 total_size -= mblk_size;
2622                                 snfv->snfv_ref++;
2623                                 mapoff = 0;
2624                                 if (i > 0)
2625                                         linkb(mp, nmp);
2626                                 else
2627                                         mp = nmp;
2628                         }
2629                         VN_HOLD(fvp);
2630                         snfv->snfv_vp = fvp;
2631                 } else {
2632                         /* vpm not supported. fallback to segmap */
2633                         snf_smap_desbinfo *snfi;
2634 
2635                         mapoff = fileoff & MAXBOFFSET;
2636                         chain_size = MAXBSIZE - mapoff;
2637                         if (chain_size > total_size)
2638                                 chain_size = total_size;
2639                         /*
2640                          * we don't forcefault because we'll call
2641                          * segmap_fault(F_SOFTLOCK) next.
2642                          *
2643                          * S_READ will get the ref bit set (by either
2644                          * segmap_getmapflt() or segmap_fault()) and page
2645                          * shared locked.
2646                          */
2647                         base = segmap_getmapflt(segkmap, fvp, fileoff,
2648                             chain_size, segmap_kpm ? SM_FAULT : 0, S_READ);
2649 
2650                         snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP);
2651                         snfi->snfi_len = (size_t)roundup(mapoff+chain_size,
2652                             PAGESIZE)- (mapoff & PAGEMASK);
2653                         /*
2654                          * We must call segmap_fault() even for segmap_kpm
2655                          * because that's how error gets returned.
2656                          * (segmap_getmapflt() never fails but segmap_fault()
2657                          * does.)
2658                          *
2659                          * If the pages aren't available yet, we get
2660                          * DEADLK, so wait and try again a little later using
2661                          * an increasing wait. We might be here a long time.
2662                          *
2663                          * If delay_sig returns EINTR, be sure to exit and
2664                          * pass it up to the caller.
2665                          */
2666                         deadlk_wait = 0;
2667                         while ((error = FC_ERRNO(segmap_fault(kas.a_hat,
2668                             segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base +
2669                             mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK,
2670                             S_READ))) == EDEADLK) {
2671                                 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2672                                 if ((error = delay_sig(deadlk_wait)) != 0) {
2673                                         break;
2674                                 }
2675                         }
2676                         if (error != 0) {
2677                                 (void) segmap_release(segkmap, base, 0);
2678                                 kmem_free(snfi, sizeof (*snfi));
2679                                 error = (error == EINTR) ? EINTR : EIO;
2680                                 goto out;
2681                         }
2682                         snfi->snfi_frtn.free_func = snf_smap_desbfree;
2683                         snfi->snfi_frtn.free_arg = (caddr_t)snfi;
2684                         snfi->snfi_base = base;
2685                         snfi->snfi_mapoff = mapoff;
2686                         mp = esballoca((uchar_t *)base + mapoff, chain_size,
2687                             BPRI_HI, &snfi->snfi_frtn);
2688 
2689                         if (mp == NULL) {
2690                                 (void) segmap_fault(kas.a_hat, segkmap,
2691                                     (caddr_t)(uintptr_t)(((uintptr_t)base +
2692                                     mapoff) & PAGEMASK), snfi->snfi_len,
2693                                     F_SOFTUNLOCK, S_OTHER);
2694                                 (void) segmap_release(segkmap, base, 0);
2695                                 kmem_free(snfi, sizeof (*snfi));
2696                                 freemsg(mp);
2697                                 error = EAGAIN;
2698                                 goto out;
2699                         }
2700                         VN_HOLD(fvp);
2701                         snfi->snfi_vp = fvp;
2702                         mp->b_wptr += chain_size;
2703 
2704                         /* Mark this dblk with the zero-copy flag */
2705                         mp->b_datap->db_struioflag |= STRUIO_ZC;
2706                         fileoff += chain_size;
2707                         total_size -= chain_size;
2708                 }
2709 
2710                 if (total_size == 0 && !nowait) {
2711                         ASSERT(!dowait);
2712                         dowait = B_TRUE;
2713                         mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
2714                 }
2715                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2716                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2717                 if (error != 0) {
2718                         /*
2719                          * mp contains the mblks that were not sent by
2720                          * socket_sendmblk. Use its size to update *count
2721                          */
2722                         *count = ksize + (chain_size - msgdsize(mp));
2723                         if (mp != NULL)
2724                                 freemsg(mp);
2725                         return (error);
2726                 }
2727                 ksize += chain_size;
2728                 if (total_size == 0)
2729                         goto done;
2730 
2731                 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2732                 va.va_mask = AT_SIZE;
2733                 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2734                 if (error)
2735                         break;
2736                 /* Read as much as possible. */
2737                 if (fileoff >= va.va_size)
2738                         break;
2739                 if (total_size + fileoff > va.va_size)
2740                         total_size = va.va_size - fileoff;
2741         }
2742 out:
2743         VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2744 done:
2745         *count = ksize;
2746         if (dowait) {
2747                 stdata_t *stp;
2748 
2749                 stp = vp->v_stream;
2750                 if (stp == NULL) {
2751                         struct sonode *so;
2752                         so = VTOSO(vp);
2753                         error = so_zcopy_wait(so);
2754                 } else {
2755                         mutex_enter(&stp->sd_lock);
2756                         while (!(stp->sd_flag & STZCNOTIFY)) {
2757                                 if (cv_wait_sig(&stp->sd_zcopy_wait,
2758                                     &stp->sd_lock) == 0) {
2759                                         error = EINTR;
2760                                         break;
2761                                 }
2762                         }
2763                         stp->sd_flag &= ~STZCNOTIFY;
2764                         mutex_exit(&stp->sd_lock);
2765                 }
2766         }
2767         return (error);
2768 }
2769 
2770 int
2771 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size,
2772     uint_t maxpsz, ssize_t *count)
2773 {
2774         struct vnode *vp;
2775         mblk_t *mp;
2776         int iosize;
2777         int extra = 0;
2778         int error;
2779         short fflag;
2780         int ksize;
2781         int ioflag;
2782         struct uio auio;
2783         struct iovec aiov;
2784         struct vattr va;
2785         int maxblk = 0;
2786         int wroff = 0;
2787         struct sonode *so;
2788         struct nmsghdr msg;
2789 
2790         vp = fp->f_vnode;
2791         if (vp->v_type == VSOCK) {
2792                 stdata_t *stp;
2793 
2794                 /*
2795                  * Get the extra space to insert a header and a trailer.
2796                  */
2797                 so = VTOSO(vp);
2798                 stp = vp->v_stream;
2799                 if (stp == NULL) {
2800                         wroff = so->so_proto_props.sopp_wroff;
2801                         maxblk = so->so_proto_props.sopp_maxblk;
2802                         extra = wroff + so->so_proto_props.sopp_tail;
2803                 } else {
2804                         wroff = (int)(stp->sd_wroff);
2805                         maxblk = (int)(stp->sd_maxblk);
2806                         extra = wroff + (int)(stp->sd_tail);
2807                 }
2808         }
2809         bzero(&msg, sizeof (msg));
2810         fflag = fp->f_flag;
2811         ksize = 0;
2812         auio.uio_iov = &aiov;
2813         auio.uio_iovcnt = 1;
2814         auio.uio_segflg = UIO_SYSSPACE;
2815         auio.uio_llimit = MAXOFFSET_T;
2816         auio.uio_fmode = fflag;
2817         auio.uio_extflg = UIO_COPY_CACHED;
2818         ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC);
2819         /* If read sync is not asked for, filter sync flags */
2820         if ((ioflag & FRSYNC) == 0)
2821                 ioflag &= ~(FSYNC|FDSYNC);
2822         for (;;) {
2823                 if (ISSIG(curthread, JUSTLOOKING)) {
2824                         error = EINTR;
2825                         break;
2826                 }
2827                 iosize = (int)MIN(maxpsz, size);
2828 
2829                 /*
2830                  * Socket filters can limit the mblk size,
2831                  * so limit reads to maxblk if there are
2832                  * filters present.
2833                  */
2834                 if (vp->v_type == VSOCK &&
2835                     so->so_filter_active > 0 && maxblk != INFPSZ)
2836                         iosize = (int)MIN(iosize, maxblk);
2837 
2838                 if (is_system_labeled()) {
2839                         mp = allocb_cred(iosize + extra, CRED(),
2840                             curproc->p_pid);
2841                 } else {
2842                         mp = allocb(iosize + extra, BPRI_MED);
2843                 }
2844                 if (mp == NULL) {
2845                         error = EAGAIN;
2846                         break;
2847                 }
2848 
2849                 mp->b_rptr += wroff;
2850 
2851                 aiov.iov_base = (caddr_t)mp->b_rptr;
2852                 aiov.iov_len = iosize;
2853                 auio.uio_loffset = fileoff;
2854                 auio.uio_resid = iosize;
2855 
2856                 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL);
2857                 iosize -= auio.uio_resid;
2858 
2859                 if (error == EINTR && iosize != 0)
2860                         error = 0;
2861 
2862                 if (error != 0 || iosize == 0) {
2863                         freeb(mp);
2864                         break;
2865                 }
2866                 mp->b_wptr = mp->b_rptr + iosize;
2867 
2868                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2869 
2870                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2871 
2872                 if (error != 0) {
2873                         *count = ksize;
2874                         if (mp != NULL)
2875                                 freeb(mp);
2876                         return (error);
2877                 }
2878                 ksize += iosize;
2879                 size -= iosize;
2880                 if (size == 0)
2881                         goto done;
2882 
2883                 fileoff += iosize;
2884                 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2885                 va.va_mask = AT_SIZE;
2886                 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2887                 if (error)
2888                         break;
2889                 /* Read as much as possible. */
2890                 if (fileoff >= va.va_size)
2891                         size = 0;
2892                 else if (size + fileoff > va.va_size)
2893                         size = va.va_size - fileoff;
2894         }
2895         VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2896 done:
2897         *count = ksize;
2898         return (error);
2899 }
2900 
2901 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
2902 /*
2903  * Largefile support for 32 bit applications only.
2904  */
2905 int
2906 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv,
2907     ssize32_t *count32)
2908 {
2909         ssize32_t sfv_len;
2910         u_offset_t sfv_off, va_size;
2911         struct vnode *vp, *fvp, *realvp;
2912         struct vattr va;
2913         stdata_t *stp;
2914         ssize_t count = 0;
2915         int error = 0;
2916         boolean_t dozcopy = B_FALSE;
2917         uint_t maxpsz;
2918 
2919         sfv_len = (ssize32_t)sfv->sfv_len;
2920         if (sfv_len < 0) {
2921                 error = EINVAL;
2922                 goto out;
2923         }
2924 
2925         if (sfv_len == 0) goto out;
2926 
2927         sfv_off = (u_offset_t)sfv->sfv_off;
2928 
2929         /* Same checks as in pread */
2930         if (sfv_off > MAXOFFSET_T) {
2931                 error = EINVAL;
2932                 goto out;
2933         }
2934         if (sfv_off + sfv_len > MAXOFFSET_T)
2935                 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off);
2936 
2937         /*
2938          * There are no more checks on sfv_len. So, we cast it to
2939          * u_offset_t and share the snf_direct_io/snf_cache code between
2940          * 32 bit and 64 bit.
2941          *
2942          * TODO: should do nbl_need_check() like read()?
2943          */
2944         if (sfv_len > sendfile_max_size) {
2945                 sf_stats.ss_file_not_cached++;
2946                 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len,
2947                     &count);
2948                 goto out;
2949         }
2950         fvp = rfp->f_vnode;
2951         if (VOP_REALVP(fvp, &realvp, NULL) == 0)
2952                 fvp = realvp;
2953         /*
2954          * Grab the lock as a reader to prevent the file size
2955          * from changing underneath.
2956          */
2957         (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2958         va.va_mask = AT_SIZE;
2959         error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2960         va_size = va.va_size;
2961         if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) {
2962                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2963                 goto out;
2964         }
2965         /* Read as much as possible. */
2966         if (sfv_off + sfv_len > va_size)
2967                 sfv_len = va_size - sfv_off;
2968 
2969         vp = fp->f_vnode;
2970         stp = vp->v_stream;
2971         /*
2972          * When the NOWAIT flag is not set, we enable zero-copy only if the
2973          * transfer size is large enough. This prevents performance loss
2974          * when the caller sends the file piece by piece.
2975          */
2976         if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) ||
2977             (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) &&
2978             !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) {
2979                 uint_t copyflag;
2980                 copyflag = stp != NULL ? stp->sd_copyflag :
2981                     VTOSO(vp)->so_proto_props.sopp_zcopyflag;
2982                 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) {
2983                         int on = 1;
2984 
2985                         if (socket_setsockopt(VTOSO(vp), SOL_SOCKET,
2986                             SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0)
2987                                 dozcopy = B_TRUE;
2988                 } else {
2989                         dozcopy = copyflag & STZCVMSAFE;
2990                 }
2991         }
2992         if (dozcopy) {
2993                 sf_stats.ss_file_segmap++;
2994                 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2995                     &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0));
2996         } else {
2997                 if (vp->v_type == VSOCK && stp == NULL) {
2998                         sonode_t *so = VTOSO(vp);
2999                         maxpsz = so->so_proto_props.sopp_maxpsz;
3000                 } else if (stp != NULL) {
3001                         maxpsz = stp->sd_qn_maxpsz;
3002                 } else {
3003                         maxpsz = maxphys;
3004                 }
3005 
3006                 if (maxpsz == INFPSZ)
3007                         maxpsz = maxphys;
3008                 else
3009                         maxpsz = roundup(maxpsz, MAXBSIZE);
3010                 sf_stats.ss_file_cached++;
3011                 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len,
3012                     maxpsz, &count);
3013         }
3014 out:
3015         releasef(sfv->sfv_fd);
3016         *count32 = (ssize32_t)count;
3017         return (error);
3018 }
3019 #endif
3020 
3021 #ifdef _SYSCALL32_IMPL
3022 /*
3023  * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a
3024  * ssize_t rather than ssize32_t; see the comments above read32 for details.
3025  */
3026 
3027 ssize_t
3028 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3029 {
3030         return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3031 }
3032 
3033 ssize_t
3034 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3035     caddr32_t name, caddr32_t namelenp)
3036 {
3037         return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3038             (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp));
3039 }
3040 
3041 ssize_t
3042 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3043 {
3044         return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3045 }
3046 
3047 ssize_t
3048 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3049     caddr32_t name, socklen_t namelen)
3050 {
3051         return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3052             (void *)(uintptr_t)name, namelen));
3053 }
3054 #endif  /* _SYSCALL32_IMPL */
3055 
3056 /*
3057  * Function wrappers (mostly around the sonode switch) for
3058  * backward compatibility.
3059  */
3060 
3061 int
3062 soaccept(struct sonode *so, int fflag, struct sonode **nsop)
3063 {
3064         return (socket_accept(so, fflag, CRED(), nsop));
3065 }
3066 
3067 int
3068 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3069     int backlog, int flags)
3070 {
3071         int     error;
3072 
3073         error = socket_bind(so, name, namelen, flags, CRED());
3074         if (error == 0 && backlog != 0)
3075                 return (socket_listen(so, backlog, CRED()));
3076 
3077         return (error);
3078 }
3079 
3080 int
3081 solisten(struct sonode *so, int backlog)
3082 {
3083         return (socket_listen(so, backlog, CRED()));
3084 }
3085 
3086 int
3087 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3088     int fflag, int flags)
3089 {
3090         return (socket_connect(so, name, namelen, fflag, flags, CRED()));
3091 }
3092 
3093 int
3094 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3095 {
3096         return (socket_recvmsg(so, msg, uiop, CRED()));
3097 }
3098 
3099 int
3100 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3101 {
3102         return (socket_sendmsg(so, msg, uiop, CRED()));
3103 }
3104 
3105 int
3106 soshutdown(struct sonode *so, int how)
3107 {
3108         return (socket_shutdown(so, how, CRED()));
3109 }
3110 
3111 int
3112 sogetsockopt(struct sonode *so, int level, int option_name, void *optval,
3113     socklen_t *optlenp, int flags)
3114 {
3115         return (socket_getsockopt(so, level, option_name, optval, optlenp,
3116             flags, CRED()));
3117 }
3118 
3119 int
3120 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval,
3121     t_uscalar_t optlen)
3122 {
3123         return (socket_setsockopt(so, level, option_name, optval, optlen,
3124             CRED()));
3125 }
3126 
3127 /*
3128  * Because this is backward compatibility interface it only needs to be
3129  * able to handle the creation of TPI sockfs sockets.
3130  */
3131 struct sonode *
3132 socreate(struct sockparams *sp, int family, int type, int protocol, int version,
3133     int *errorp)
3134 {
3135         struct sonode *so;
3136 
3137         ASSERT(sp != NULL);
3138 
3139         so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol,
3140             version, SOCKET_SLEEP, errorp, CRED());
3141         if (so == NULL) {
3142                 SOCKPARAMS_DEC_REF(sp);
3143         } else {
3144                 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) {
3145                         /* Cannot fail, only bumps so_count */
3146                         (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL);
3147                 } else {
3148                         socket_destroy(so);
3149                         so = NULL;
3150                 }
3151         }
3152         return (so);
3153 }