1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 /*
31 * Portions of this source code were derived from Berkeley 4.3 BSD
32 * under license from the Regents of the University of California.
33 */
34
35 #include <sys/types.h>
36 #include <sys/t_lock.h>
37 #include <sys/param.h>
38 #include <sys/time.h>
39 #include <sys/systm.h>
40 #include <sys/sysmacros.h>
41 #include <sys/resource.h>
42 #include <sys/signal.h>
43 #include <sys/cred.h>
44 #include <sys/user.h>
45 #include <sys/buf.h>
46 #include <sys/vfs.h>
47 #include <sys/vnode.h>
48 #include <sys/proc.h>
49 #include <sys/disp.h>
50 #include <sys/file.h>
51 #include <sys/fcntl.h>
52 #include <sys/flock.h>
53 #include <sys/kmem.h>
54 #include <sys/uio.h>
55 #include <sys/dnlc.h>
56 #include <sys/conf.h>
57 #include <sys/mman.h>
58 #include <sys/pathname.h>
59 #include <sys/debug.h>
60 #include <sys/vmsystm.h>
61 #include <sys/cmn_err.h>
62 #include <sys/filio.h>
63 #include <sys/atomic.h>
64
65 #include <sys/fssnap_if.h>
66 #include <sys/fs/ufs_fs.h>
67 #include <sys/fs/ufs_lockfs.h>
68 #include <sys/fs/ufs_filio.h>
69 #include <sys/fs/ufs_inode.h>
70 #include <sys/fs/ufs_fsdir.h>
71 #include <sys/fs/ufs_quota.h>
72 #include <sys/fs/ufs_trans.h>
73 #include <sys/fs/ufs_panic.h>
74 #include <sys/dirent.h> /* must be AFTER <sys/fs/fsdir.h>! */
75 #include <sys/errno.h>
76
77 #include <sys/filio.h> /* _FIOIO */
78
79 #include <vm/hat.h>
80 #include <vm/page.h>
81 #include <vm/pvn.h>
82 #include <vm/as.h>
83 #include <vm/seg.h>
84 #include <vm/seg_map.h>
85 #include <vm/seg_vn.h>
86 #include <vm/seg_kmem.h>
87 #include <vm/rm.h>
88 #include <sys/swap.h>
89 #include <sys/epm.h>
90
91 #include <fs/fs_subr.h>
92
93 static void *ufs_directio_zero_buf;
94 static int ufs_directio_zero_len = 8192;
95
96 int ufs_directio_enabled = 1; /* feature is enabled */
97
98 /*
99 * for kstats reader
100 */
101 struct ufs_directio_kstats {
102 kstat_named_t logical_reads;
103 kstat_named_t phys_reads;
104 kstat_named_t hole_reads;
105 kstat_named_t nread;
106 kstat_named_t logical_writes;
107 kstat_named_t phys_writes;
108 kstat_named_t nwritten;
109 kstat_named_t nflushes;
110 } ufs_directio_kstats = {
111 { "logical_reads", KSTAT_DATA_UINT64 },
112 { "phys_reads", KSTAT_DATA_UINT64 },
113 { "hole_reads", KSTAT_DATA_UINT64 },
114 { "nread", KSTAT_DATA_UINT64 },
115 { "logical_writes", KSTAT_DATA_UINT64 },
116 { "phys_writes", KSTAT_DATA_UINT64 },
117 { "nwritten", KSTAT_DATA_UINT64 },
118 { "nflushes", KSTAT_DATA_UINT64 },
119 };
120
121 kstat_t *ufs_directio_kstatsp;
122
123 /*
124 * use kmem_cache_create for direct-physio buffers. This has shown
125 * a better cache distribution compared to buffers on the
126 * stack. It also avoids semaphore construction/deconstruction
127 * per request
128 */
129 struct directio_buf {
130 struct directio_buf *next;
131 char *addr;
132 size_t nbytes;
133 struct buf buf;
134 };
135 static struct kmem_cache *directio_buf_cache;
136
137
138 /* ARGSUSED */
139 static int
140 directio_buf_constructor(void *dbp, void *cdrarg, int kmflags)
141 {
142 bioinit((struct buf *)&((struct directio_buf *)dbp)->buf);
143 return (0);
144 }
145
146 /* ARGSUSED */
147 static void
148 directio_buf_destructor(void *dbp, void *cdrarg)
149 {
150 biofini((struct buf *)&((struct directio_buf *)dbp)->buf);
151 }
152
153 void
154 directio_bufs_init(void)
155 {
156 directio_buf_cache = kmem_cache_create("directio_buf_cache",
157 sizeof (struct directio_buf), 0,
158 directio_buf_constructor, directio_buf_destructor,
159 NULL, NULL, NULL, 0);
160 }
161
162 void
163 ufs_directio_init(void)
164 {
165 /*
166 * kstats
167 */
168 ufs_directio_kstatsp = kstat_create("ufs", 0,
169 "directio", "ufs", KSTAT_TYPE_NAMED,
170 sizeof (ufs_directio_kstats) / sizeof (kstat_named_t),
171 KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE);
172 if (ufs_directio_kstatsp) {
173 ufs_directio_kstatsp->ks_data = (void *)&ufs_directio_kstats;
174 kstat_install(ufs_directio_kstatsp);
175 }
176 /*
177 * kzero is broken so we have to use a private buf of zeroes
178 */
179 ufs_directio_zero_buf = kmem_zalloc(ufs_directio_zero_len, KM_SLEEP);
180 directio_bufs_init();
181 }
182
183 /*
184 * Wait for the first direct IO operation to finish
185 */
186 static int
187 directio_wait_one(struct directio_buf *dbp, long *bytes_iop)
188 {
189 buf_t *bp;
190 int error;
191
192 /*
193 * Wait for IO to finish
194 */
195 bp = &dbp->buf;
196 error = biowait(bp);
197
198 /*
199 * bytes_io will be used to figure out a resid
200 * for the caller. The resid is approximated by reporting
201 * the bytes following the first failed IO as the residual.
202 *
203 * I am cautious about using b_resid because I
204 * am not sure how well the disk drivers maintain it.
205 */
206 if (error)
207 if (bp->b_resid)
208 *bytes_iop = bp->b_bcount - bp->b_resid;
209 else
210 *bytes_iop = 0;
211 else
212 *bytes_iop += bp->b_bcount;
213 /*
214 * Release direct IO resources
215 */
216 bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_SHADOW);
217 kmem_cache_free(directio_buf_cache, dbp);
218 return (error);
219 }
220
221 /*
222 * Wait for all of the direct IO operations to finish
223 */
224
225 static int
226 directio_wait(struct directio_buf *tail, long *bytes_iop)
227 {
228 int error = 0, newerror;
229 struct directio_buf *dbp;
230
231 /*
232 * The linked list of directio buf structures is maintained
233 * in reverse order (tail->last request->penultimate request->...)
234 */
235 while ((dbp = tail) != NULL) {
236 tail = dbp->next;
237 newerror = directio_wait_one(dbp, bytes_iop);
238 if (error == 0)
239 error = newerror;
240 }
241 return (error);
242 }
243 /*
244 * Initiate direct IO request
245 */
246 static void
247 directio_start(struct ufsvfs *ufsvfsp, struct inode *ip, size_t nbytes,
248 offset_t offset, char *addr, enum seg_rw rw, struct proc *procp,
249 struct directio_buf **tailp, page_t **pplist)
250 {
251 buf_t *bp;
252 struct directio_buf *dbp;
253
254 /*
255 * Allocate a directio buf header
256 * Note - list is maintained in reverse order.
257 * directio_wait_one() depends on this fact when
258 * adjusting the ``bytes_io'' param. bytes_io
259 * is used to compute a residual in the case of error.
260 */
261 dbp = kmem_cache_alloc(directio_buf_cache, KM_SLEEP);
262 dbp->next = *tailp;
263 *tailp = dbp;
264
265 /*
266 * Initialize buf header
267 */
268 dbp->addr = addr;
269 dbp->nbytes = nbytes;
270 bp = &dbp->buf;
271 bp->b_edev = ip->i_dev;
272 bp->b_lblkno = btodt(offset);
273 bp->b_bcount = nbytes;
274 bp->b_un.b_addr = addr;
275 bp->b_proc = procp;
276 bp->b_file = ip->i_vnode;
277
278 /*
279 * Note that S_WRITE implies B_READ and vice versa: a read(2)
280 * will B_READ data from the filesystem and S_WRITE it into
281 * the user's buffer; a write(2) will S_READ data from the
282 * user's buffer and B_WRITE it to the filesystem.
283 */
284 if (rw == S_WRITE) {
285 bp->b_flags = B_BUSY | B_PHYS | B_READ;
286 ufs_directio_kstats.phys_reads.value.ui64++;
287 ufs_directio_kstats.nread.value.ui64 += nbytes;
288 } else {
289 bp->b_flags = B_BUSY | B_PHYS | B_WRITE;
290 ufs_directio_kstats.phys_writes.value.ui64++;
291 ufs_directio_kstats.nwritten.value.ui64 += nbytes;
292 }
293 bp->b_shadow = pplist;
294 if (pplist != NULL)
295 bp->b_flags |= B_SHADOW;
296
297 /*
298 * Issue I/O request.
299 */
300 ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
301 if (ufsvfsp->vfs_snapshot)
302 fssnap_strategy(&ufsvfsp->vfs_snapshot, bp);
303 else
304 (void) bdev_strategy(bp);
305
306 if (rw == S_WRITE)
307 lwp_stat_update(LWP_STAT_OUBLK, 1);
308 else
309 lwp_stat_update(LWP_STAT_INBLK, 1);
310
311 }
312
313 uint32_t ufs_shared_writes; /* writes done w/ lock shared */
314 uint32_t ufs_cur_writes; /* # concurrent writes */
315 uint32_t ufs_maxcur_writes; /* high water concurrent writes */
316 uint32_t ufs_posix_hits; /* writes done /w lock excl. */
317
318 /*
319 * Force POSIX syncronous data integrity on all writes for testing.
320 */
321 uint32_t ufs_force_posix_sdi = 0;
322
323 /*
324 * Direct Write
325 */
326
327 int
328 ufs_directio_write(struct inode *ip, uio_t *arg_uio, int ioflag, int rewrite,
329 cred_t *cr, int *statusp)
330 {
331 long resid, bytes_written;
332 u_offset_t size, uoff;
333 uio_t *uio = arg_uio;
334 rlim64_t limit = uio->uio_llimit;
335 int on, n, error, newerror, len, has_holes;
336 daddr_t bn;
337 size_t nbytes;
338 struct fs *fs;
339 vnode_t *vp;
340 iovec_t *iov;
341 struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
342 struct proc *procp;
343 struct as *as;
344 struct directio_buf *tail;
345 int exclusive, ncur, bmap_peek;
346 uio_t copy_uio;
347 iovec_t copy_iov;
348 char *copy_base;
349 long copy_resid;
350
351 /*
352 * assume that directio isn't possible (normal case)
353 */
354 *statusp = DIRECTIO_FAILURE;
355
356 /*
357 * Don't go direct
358 */
359 if (ufs_directio_enabled == 0)
360 return (0);
361
362 /*
363 * mapped file; nevermind
364 */
365 if (ip->i_mapcnt)
366 return (0);
367
368 /*
369 * CAN WE DO DIRECT IO?
370 */
371 uoff = uio->uio_loffset;
372 resid = uio->uio_resid;
373
374 /*
375 * beyond limit
376 */
377 if (uoff + resid > limit)
378 return (0);
379
380 /*
381 * must be sector aligned
382 */
383 if ((uoff & (u_offset_t)(DEV_BSIZE - 1)) || (resid & (DEV_BSIZE - 1)))
384 return (0);
385
386 /*
387 * SHOULD WE DO DIRECT IO?
388 */
389 size = ip->i_size;
390 has_holes = -1;
391
392 /*
393 * only on regular files; no metadata
394 */
395 if (((ip->i_mode & IFMT) != IFREG) || ip->i_ufsvfs->vfs_qinod == ip)
396 return (0);
397
398 /*
399 * Synchronous, allocating writes run very slow in Direct-Mode
400 * XXX - can be fixed with bmap_write changes for large writes!!!
401 * XXX - can be fixed for updates to "almost-full" files
402 * XXX - WARNING - system hangs if bmap_write() has to
403 * allocate lots of pages since pageout
404 * suspends on locked inode
405 */
406 if (!rewrite && (ip->i_flag & ISYNC)) {
407 if ((uoff + resid) > size)
408 return (0);
409 has_holes = bmap_has_holes(ip);
410 if (has_holes)
411 return (0);
412 }
413
414 /*
415 * Each iovec must be short aligned and sector aligned. If
416 * one is not, then kmem_alloc a new buffer and copy all of
417 * the smaller buffers into the new buffer. This new
418 * buffer will be short aligned and sector aligned.
419 */
420 iov = uio->uio_iov;
421 nbytes = uio->uio_iovcnt;
422 while (nbytes--) {
423 if (((uint_t)iov->iov_len & (DEV_BSIZE - 1)) != 0 ||
424 (intptr_t)(iov->iov_base) & 1) {
425 copy_resid = uio->uio_resid;
426 copy_base = kmem_alloc(copy_resid, KM_NOSLEEP);
427 if (copy_base == NULL)
428 return (0);
429 copy_iov.iov_base = copy_base;
430 copy_iov.iov_len = copy_resid;
431 copy_uio.uio_iov = ©_iov;
432 copy_uio.uio_iovcnt = 1;
433 copy_uio.uio_segflg = UIO_SYSSPACE;
434 copy_uio.uio_extflg = UIO_COPY_DEFAULT;
435 copy_uio.uio_loffset = uio->uio_loffset;
436 copy_uio.uio_resid = uio->uio_resid;
437 copy_uio.uio_llimit = uio->uio_llimit;
438 error = uiomove(copy_base, copy_resid, UIO_WRITE, uio);
439 if (error) {
440 kmem_free(copy_base, copy_resid);
441 return (0);
442 }
443 uio = ©_uio;
444 break;
445 }
446 iov++;
447 }
448
449 /*
450 * From here on down, all error exits must go to errout and
451 * not simply return a 0.
452 */
453
454 /*
455 * DIRECTIO
456 */
457
458 fs = ip->i_fs;
459
460 /*
461 * POSIX check. If attempting a concurrent re-write, make sure
462 * that this will be a single request to the driver to meet
463 * POSIX synchronous data integrity requirements.
464 */
465 bmap_peek = 0;
466 if (rewrite && ((ioflag & FDSYNC) || ufs_force_posix_sdi)) {
467 int upgrade = 0;
468
469 /* check easy conditions first */
470 if (uio->uio_iovcnt != 1 || resid > ufsvfsp->vfs_ioclustsz) {
471 upgrade = 1;
472 } else {
473 /* now look for contiguous allocation */
474 len = (ssize_t)blkroundup(fs, resid);
475 error = bmap_read(ip, uoff, &bn, &len);
476 if (error || bn == UFS_HOLE || len == 0)
477 goto errout;
478 /* save a call to bmap_read later */
479 bmap_peek = 1;
480 if (len < resid)
481 upgrade = 1;
482 }
483 if (upgrade) {
484 rw_exit(&ip->i_contents);
485 rw_enter(&ip->i_contents, RW_WRITER);
486 ufs_posix_hits++;
487 }
488 }
489
490
491 /*
492 * allocate space
493 */
494
495 /*
496 * If attempting a re-write, there is no allocation to do.
497 * bmap_write would trip an ASSERT if i_contents is held shared.
498 */
499 if (rewrite)
500 goto skip_alloc;
501
502 do {
503 on = (int)blkoff(fs, uoff);
504 n = (int)MIN(fs->fs_bsize - on, resid);
505 if ((uoff + n) > ip->i_size) {
506 error = bmap_write(ip, uoff, (int)(on + n),
507 (int)(uoff & (offset_t)MAXBOFFSET) == 0,
508 NULL, cr);
509 /* Caller is responsible for updating i_seq if needed */
510 if (error)
511 break;
512 ip->i_size = uoff + n;
513 ip->i_flag |= IATTCHG;
514 } else if (n == MAXBSIZE) {
515 error = bmap_write(ip, uoff, (int)(on + n),
516 BI_ALLOC_ONLY, NULL, cr);
517 /* Caller is responsible for updating i_seq if needed */
518 } else {
519 if (has_holes < 0)
520 has_holes = bmap_has_holes(ip);
521 if (has_holes) {
522 uint_t blk_size;
523 u_offset_t offset;
524
525 offset = uoff & (offset_t)fs->fs_bmask;
526 blk_size = (int)blksize(fs, ip,
527 (daddr_t)lblkno(fs, offset));
528 error = bmap_write(ip, uoff, blk_size,
529 BI_NORMAL, NULL, cr);
530 /*
531 * Caller is responsible for updating
532 * i_seq if needed
533 */
534 } else
535 error = 0;
536 }
537 if (error)
538 break;
539 uoff += n;
540 resid -= n;
541 /*
542 * if file has grown larger than 2GB, set flag
543 * in superblock if not already set
544 */
545 if ((ip->i_size > MAXOFF32_T) &&
546 !(fs->fs_flags & FSLARGEFILES)) {
547 ASSERT(ufsvfsp->vfs_lfflags & UFS_LARGEFILES);
548 mutex_enter(&ufsvfsp->vfs_lock);
549 fs->fs_flags |= FSLARGEFILES;
550 ufs_sbwrite(ufsvfsp);
551 mutex_exit(&ufsvfsp->vfs_lock);
552 }
553 } while (resid);
554
555 if (error) {
556 /*
557 * restore original state
558 */
559 if (resid) {
560 if (size == ip->i_size)
561 goto errout;
562 (void) ufs_itrunc(ip, size, 0, cr);
563 }
564 /*
565 * try non-directio path
566 */
567 goto errout;
568 }
569 skip_alloc:
570
571 /*
572 * get rid of cached pages
573 */
574 vp = ITOV(ip);
575 exclusive = rw_write_held(&ip->i_contents);
576 if (vn_has_cached_data(vp)) {
577 if (!exclusive) {
578 /*
579 * Still holding i_rwlock, so no allocations
580 * can happen after dropping contents.
581 */
582 rw_exit(&ip->i_contents);
583 rw_enter(&ip->i_contents, RW_WRITER);
584 }
585 (void) VOP_PUTPAGE(vp, (offset_t)0, (size_t)0,
586 B_INVAL, cr, NULL);
587 if (vn_has_cached_data(vp))
588 goto errout;
589 if (!exclusive)
590 rw_downgrade(&ip->i_contents);
591 ufs_directio_kstats.nflushes.value.ui64++;
592 }
593
594 /*
595 * Direct Writes
596 */
597
598 if (!exclusive) {
599 ufs_shared_writes++;
600 ncur = atomic_inc_32_nv(&ufs_cur_writes);
601 if (ncur > ufs_maxcur_writes)
602 ufs_maxcur_writes = ncur;
603 }
604
605 /*
606 * proc and as are for VM operations in directio_start()
607 */
608 if (uio->uio_segflg == UIO_USERSPACE) {
609 procp = ttoproc(curthread);
610 as = procp->p_as;
611 } else {
612 procp = NULL;
613 as = &kas;
614 }
615 *statusp = DIRECTIO_SUCCESS;
616 error = 0;
617 newerror = 0;
618 resid = uio->uio_resid;
619 bytes_written = 0;
620 ufs_directio_kstats.logical_writes.value.ui64++;
621 while (error == 0 && newerror == 0 && resid && uio->uio_iovcnt) {
622 size_t pglck_len, pglck_size;
623 caddr_t pglck_base;
624 page_t **pplist, **spplist;
625
626 tail = NULL;
627
628 /*
629 * Adjust number of bytes
630 */
631 iov = uio->uio_iov;
632 pglck_len = (size_t)MIN(iov->iov_len, resid);
633 pglck_base = iov->iov_base;
634 if (pglck_len == 0) {
635 uio->uio_iov++;
636 uio->uio_iovcnt--;
637 continue;
638 }
639
640 /*
641 * Try to Lock down the largest chunck of pages possible.
642 */
643 pglck_len = (size_t)MIN(pglck_len, ufsvfsp->vfs_ioclustsz);
644 error = as_pagelock(as, &pplist, pglck_base, pglck_len, S_READ);
645
646 if (error)
647 break;
648
649 pglck_size = pglck_len;
650 while (pglck_len) {
651
652 nbytes = pglck_len;
653 uoff = uio->uio_loffset;
654
655 if (!bmap_peek) {
656
657 /*
658 * Re-adjust number of bytes to contiguous
659 * range. May have already called bmap_read
660 * in the case of a concurrent rewrite.
661 */
662 len = (ssize_t)blkroundup(fs, nbytes);
663 error = bmap_read(ip, uoff, &bn, &len);
664 if (error)
665 break;
666 if (bn == UFS_HOLE || len == 0)
667 break;
668 }
669 nbytes = (size_t)MIN(nbytes, len);
670 bmap_peek = 0;
671
672 /*
673 * Get the pagelist pointer for this offset to be
674 * passed to directio_start.
675 */
676
677 if (pplist != NULL)
678 spplist = pplist +
679 btop((uintptr_t)iov->iov_base -
680 ((uintptr_t)pglck_base & PAGEMASK));
681 else
682 spplist = NULL;
683
684 /*
685 * Kick off the direct write requests
686 */
687 directio_start(ufsvfsp, ip, nbytes, ldbtob(bn),
688 iov->iov_base, S_READ, procp, &tail, spplist);
689
690 /*
691 * Adjust pointers and counters
692 */
693 iov->iov_len -= nbytes;
694 iov->iov_base += nbytes;
695 uio->uio_loffset += nbytes;
696 resid -= nbytes;
697 pglck_len -= nbytes;
698 }
699
700 /*
701 * Wait for outstanding requests
702 */
703 newerror = directio_wait(tail, &bytes_written);
704
705 /*
706 * Release VM resources
707 */
708 as_pageunlock(as, pplist, pglck_base, pglck_size, S_READ);
709
710 }
711
712 if (!exclusive) {
713 atomic_dec_32(&ufs_cur_writes);
714 /*
715 * If this write was done shared, readers may
716 * have pulled in unmodified pages. Get rid of
717 * these potentially stale pages.
718 */
719 if (vn_has_cached_data(vp)) {
720 rw_exit(&ip->i_contents);
721 rw_enter(&ip->i_contents, RW_WRITER);
722 (void) VOP_PUTPAGE(vp, (offset_t)0, (size_t)0,
723 B_INVAL, cr, NULL);
724 ufs_directio_kstats.nflushes.value.ui64++;
725 rw_downgrade(&ip->i_contents);
726 }
727 }
728
729 /*
730 * If error, adjust resid to begin at the first
731 * un-writable byte.
732 */
733 if (error == 0)
734 error = newerror;
735 if (error)
736 resid = uio->uio_resid - bytes_written;
737 arg_uio->uio_resid = resid;
738
739 if (!rewrite) {
740 ip->i_flag |= IUPD | ICHG;
741 /* Caller will update i_seq */
742 TRANS_INODE(ip->i_ufsvfs, ip);
743 }
744 /*
745 * If there is a residual; adjust the EOF if necessary
746 */
747 if (resid) {
748 if (size != ip->i_size) {
749 if (uio->uio_loffset > size)
750 size = uio->uio_loffset;
751 (void) ufs_itrunc(ip, size, 0, cr);
752 }
753 }
754
755 if (uio == ©_uio)
756 kmem_free(copy_base, copy_resid);
757
758 return (error);
759
760 errout:
761 if (uio == ©_uio)
762 kmem_free(copy_base, copy_resid);
763
764 return (0);
765 }
766 /*
767 * Direct read of a hole
768 */
769 static int
770 directio_hole(struct uio *uio, size_t nbytes)
771 {
772 int error = 0, nzero;
773 uio_t phys_uio;
774 iovec_t phys_iov;
775
776 ufs_directio_kstats.hole_reads.value.ui64++;
777 ufs_directio_kstats.nread.value.ui64 += nbytes;
778
779 phys_iov.iov_base = uio->uio_iov->iov_base;
780 phys_iov.iov_len = nbytes;
781
782 phys_uio.uio_iov = &phys_iov;
783 phys_uio.uio_iovcnt = 1;
784 phys_uio.uio_resid = phys_iov.iov_len;
785 phys_uio.uio_segflg = uio->uio_segflg;
786 phys_uio.uio_extflg = uio->uio_extflg;
787 while (error == 0 && phys_uio.uio_resid) {
788 nzero = (int)MIN(phys_iov.iov_len, ufs_directio_zero_len);
789 error = uiomove(ufs_directio_zero_buf, nzero, UIO_READ,
790 &phys_uio);
791 }
792 return (error);
793 }
794
795 /*
796 * Direct Read
797 */
798 int
799 ufs_directio_read(struct inode *ip, uio_t *uio, cred_t *cr, int *statusp)
800 {
801 ssize_t resid, bytes_read;
802 u_offset_t size, uoff;
803 int error, newerror, len;
804 size_t nbytes;
805 struct fs *fs;
806 vnode_t *vp;
807 daddr_t bn;
808 iovec_t *iov;
809 struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
810 struct proc *procp;
811 struct as *as;
812 struct directio_buf *tail;
813
814 /*
815 * assume that directio isn't possible (normal case)
816 */
817 *statusp = DIRECTIO_FAILURE;
818
819 /*
820 * Don't go direct
821 */
822 if (ufs_directio_enabled == 0)
823 return (0);
824
825 /*
826 * mapped file; nevermind
827 */
828 if (ip->i_mapcnt)
829 return (0);
830
831 /*
832 * CAN WE DO DIRECT IO?
833 */
834 /*
835 * must be sector aligned
836 */
837 uoff = uio->uio_loffset;
838 resid = uio->uio_resid;
839 if ((uoff & (u_offset_t)(DEV_BSIZE - 1)) || (resid & (DEV_BSIZE - 1)))
840 return (0);
841 /*
842 * must be short aligned and sector aligned
843 */
844 iov = uio->uio_iov;
845 nbytes = uio->uio_iovcnt;
846 while (nbytes--) {
847 if (((size_t)iov->iov_len & (DEV_BSIZE - 1)) != 0)
848 return (0);
849 if ((intptr_t)(iov++->iov_base) & 1)
850 return (0);
851 }
852
853 /*
854 * DIRECTIO
855 */
856 fs = ip->i_fs;
857
858 /*
859 * don't read past EOF
860 */
861 size = ip->i_size;
862
863 /*
864 * The file offset is past EOF so bail out here; we don't want
865 * to update uio_resid and make it look like we read something.
866 * We say that direct I/O was a success to avoid having rdip()
867 * go through the same "read past EOF logic".
868 */
869 if (uoff >= size) {
870 *statusp = DIRECTIO_SUCCESS;
871 return (0);
872 }
873
874 /*
875 * The read would extend past EOF so make it smaller.
876 */
877 if ((uoff + resid) > size) {
878 resid = size - uoff;
879 /*
880 * recheck sector alignment
881 */
882 if (resid & (DEV_BSIZE - 1))
883 return (0);
884 }
885
886 /*
887 * At this point, we know there is some real work to do.
888 */
889 ASSERT(resid);
890
891 /*
892 * get rid of cached pages
893 */
894 vp = ITOV(ip);
895 if (vn_has_cached_data(vp)) {
896 rw_exit(&ip->i_contents);
897 rw_enter(&ip->i_contents, RW_WRITER);
898 (void) VOP_PUTPAGE(vp, (offset_t)0, (size_t)0,
899 B_INVAL, cr, NULL);
900 if (vn_has_cached_data(vp))
901 return (0);
902 rw_downgrade(&ip->i_contents);
903 ufs_directio_kstats.nflushes.value.ui64++;
904 }
905 /*
906 * Direct Reads
907 */
908
909 /*
910 * proc and as are for VM operations in directio_start()
911 */
912 if (uio->uio_segflg == UIO_USERSPACE) {
913 procp = ttoproc(curthread);
914 as = procp->p_as;
915 } else {
916 procp = NULL;
917 as = &kas;
918 }
919
920 *statusp = DIRECTIO_SUCCESS;
921 error = 0;
922 newerror = 0;
923 bytes_read = 0;
924 ufs_directio_kstats.logical_reads.value.ui64++;
925 while (error == 0 && newerror == 0 && resid && uio->uio_iovcnt) {
926 size_t pglck_len, pglck_size;
927 caddr_t pglck_base;
928 page_t **pplist, **spplist;
929
930 tail = NULL;
931
932 /*
933 * Adjust number of bytes
934 */
935 iov = uio->uio_iov;
936 pglck_len = (size_t)MIN(iov->iov_len, resid);
937 pglck_base = iov->iov_base;
938 if (pglck_len == 0) {
939 uio->uio_iov++;
940 uio->uio_iovcnt--;
941 continue;
942 }
943
944 /*
945 * Try to Lock down the largest chunck of pages possible.
946 */
947 pglck_len = (size_t)MIN(pglck_len, ufsvfsp->vfs_ioclustsz);
948 error = as_pagelock(as, &pplist, pglck_base,
949 pglck_len, S_WRITE);
950
951 if (error)
952 break;
953
954 pglck_size = pglck_len;
955 while (pglck_len) {
956
957 nbytes = pglck_len;
958 uoff = uio->uio_loffset;
959
960 /*
961 * Re-adjust number of bytes to contiguous range
962 */
963 len = (ssize_t)blkroundup(fs, nbytes);
964 error = bmap_read(ip, uoff, &bn, &len);
965 if (error)
966 break;
967
968 if (bn == UFS_HOLE) {
969 nbytes = (size_t)MIN(fs->fs_bsize -
970 (long)blkoff(fs, uoff), nbytes);
971 error = directio_hole(uio, nbytes);
972 /*
973 * Hole reads are not added to the list
974 * processed by directio_wait() below so
975 * account for bytes read here.
976 */
977 if (!error)
978 bytes_read += nbytes;
979 } else {
980 nbytes = (size_t)MIN(nbytes, len);
981
982 /*
983 * Get the pagelist pointer for this offset
984 * to be passed to directio_start.
985 */
986 if (pplist != NULL)
987 spplist = pplist +
988 btop((uintptr_t)iov->iov_base -
989 ((uintptr_t)pglck_base & PAGEMASK));
990 else
991 spplist = NULL;
992
993 /*
994 * Kick off the direct read requests
995 */
996 directio_start(ufsvfsp, ip, nbytes,
997 ldbtob(bn), iov->iov_base,
998 S_WRITE, procp, &tail, spplist);
999 }
1000
1001 if (error)
1002 break;
1003
1004 /*
1005 * Adjust pointers and counters
1006 */
1007 iov->iov_len -= nbytes;
1008 iov->iov_base += nbytes;
1009 uio->uio_loffset += nbytes;
1010 resid -= nbytes;
1011 pglck_len -= nbytes;
1012 }
1013
1014 /*
1015 * Wait for outstanding requests
1016 */
1017 newerror = directio_wait(tail, &bytes_read);
1018 /*
1019 * Release VM resources
1020 */
1021 as_pageunlock(as, pplist, pglck_base, pglck_size, S_WRITE);
1022
1023 }
1024
1025 /*
1026 * If error, adjust resid to begin at the first
1027 * un-read byte.
1028 */
1029 if (error == 0)
1030 error = newerror;
1031 uio->uio_resid -= bytes_read;
1032 return (error);
1033 }