1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 /*
29 * Vnode operations for the High Sierra filesystem
30 */
31
32 #include <sys/types.h>
33 #include <sys/t_lock.h>
34 #include <sys/param.h>
35 #include <sys/time.h>
36 #include <sys/systm.h>
37 #include <sys/sysmacros.h>
38 #include <sys/resource.h>
39 #include <sys/signal.h>
40 #include <sys/cred.h>
41 #include <sys/user.h>
42 #include <sys/buf.h>
43 #include <sys/vfs.h>
44 #include <sys/vfs_opreg.h>
45 #include <sys/stat.h>
46 #include <sys/vnode.h>
47 #include <sys/mode.h>
48 #include <sys/proc.h>
49 #include <sys/disp.h>
50 #include <sys/file.h>
51 #include <sys/fcntl.h>
52 #include <sys/flock.h>
53 #include <sys/kmem.h>
54 #include <sys/uio.h>
55 #include <sys/conf.h>
56 #include <sys/errno.h>
57 #include <sys/mman.h>
58 #include <sys/pathname.h>
59 #include <sys/debug.h>
60 #include <sys/vmsystm.h>
61 #include <sys/cmn_err.h>
62 #include <sys/fbuf.h>
63 #include <sys/dirent.h>
64 #include <sys/errno.h>
65 #include <sys/dkio.h>
66 #include <sys/cmn_err.h>
67 #include <sys/atomic.h>
68
69 #include <vm/hat.h>
70 #include <vm/page.h>
71 #include <vm/pvn.h>
72 #include <vm/as.h>
73 #include <vm/seg.h>
74 #include <vm/seg_map.h>
75 #include <vm/seg_kmem.h>
76 #include <vm/seg_vn.h>
77 #include <vm/rm.h>
78 #include <vm/page.h>
79 #include <sys/swap.h>
80 #include <sys/avl.h>
81 #include <sys/sunldi.h>
82 #include <sys/ddi.h>
83 #include <sys/sunddi.h>
84 #include <sys/sdt.h>
85
86 /*
87 * For struct modlinkage
88 */
89 #include <sys/modctl.h>
90
91 #include <sys/fs/hsfs_spec.h>
92 #include <sys/fs/hsfs_node.h>
93 #include <sys/fs/hsfs_impl.h>
94 #include <sys/fs/hsfs_susp.h>
95 #include <sys/fs/hsfs_rrip.h>
96
97 #include <fs/fs_subr.h>
98
99 /* # of contiguous requests to detect sequential access pattern */
100 static int seq_contig_requests = 2;
101
102 /*
103 * This is the max number os taskq threads that will be created
104 * if required. Since we are using a Dynamic TaskQ by default only
105 * one thread is created initially.
106 *
107 * NOTE: In the usual hsfs use case this per fs instance number
108 * of taskq threads should not place any undue load on a system.
109 * Even on an unusual system with say 100 CDROM drives, 800 threads
110 * will not be created unless all the drives are loaded and all
111 * of them are saturated with I/O at the same time! If there is at
112 * all a complaint of system load due to such an unusual case it
113 * should be easy enough to change to one per-machine Dynamic TaskQ
114 * for all hsfs mounts with a nthreads of say 32.
115 */
116 static int hsfs_taskq_nthreads = 8; /* # of taskq threads per fs */
117
118 /* Min count of adjacent bufs that will avoid buf coalescing */
119 static int hsched_coalesce_min = 2;
120
121 /*
122 * Kmem caches for heavily used small allocations. Using these kmem
123 * caches provides a factor of 3 reduction in system time and greatly
124 * aids overall throughput esp. on SPARC.
125 */
126 struct kmem_cache *hio_cache;
127 struct kmem_cache *hio_info_cache;
128
129 /*
130 * This tunable allows us to ignore inode numbers from rrip-1.12.
131 * In this case, we fall back to our default inode algorithm.
132 */
133 extern int use_rrip_inodes;
134
135 /*
136 * Free behind logic from UFS to tame our thirst for
137 * the page cache.
138 * See usr/src/uts/common/fs/ufs/ufs_vnops.c for more
139 * explanation.
140 */
141 static int freebehind = 1;
142 static int smallfile = 0;
143 static int cache_read_ahead = 0;
144 static u_offset_t smallfile64 = 32 * 1024;
145 #define SMALLFILE1_D 1000
146 #define SMALLFILE2_D 10
147 static u_offset_t smallfile1 = 32 * 1024;
148 static u_offset_t smallfile2 = 32 * 1024;
149 static clock_t smallfile_update = 0; /* when to recompute */
150 static uint_t smallfile1_d = SMALLFILE1_D;
151 static uint_t smallfile2_d = SMALLFILE2_D;
152
153 static int hsched_deadline_compare(const void *x1, const void *x2);
154 static int hsched_offset_compare(const void *x1, const void *x2);
155 static void hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra);
156 int hsched_invoke_strategy(struct hsfs *fsp);
157
158 /* ARGSUSED */
159 static int
160 hsfs_fsync(vnode_t *cp,
161 int syncflag,
162 cred_t *cred,
163 caller_context_t *ct)
164 {
165 return (0);
166 }
167
168
169 /*ARGSUSED*/
170 static int
171 hsfs_read(struct vnode *vp,
172 struct uio *uiop,
173 int ioflag,
174 struct cred *cred,
175 struct caller_context *ct)
176 {
177 caddr_t base;
178 offset_t diff;
179 int error;
180 struct hsnode *hp;
181 uint_t filesize;
182 int dofree;
183
184 hp = VTOH(vp);
185 /*
186 * if vp is of type VDIR, make sure dirent
187 * is filled up with all info (because of ptbl)
188 */
189 if (vp->v_type == VDIR) {
190 if (hp->hs_dirent.ext_size == 0)
191 hs_filldirent(vp, &hp->hs_dirent);
192 }
193 filesize = hp->hs_dirent.ext_size;
194
195 /* Sanity checks. */
196 if (uiop->uio_resid == 0 || /* No data wanted. */
197 uiop->uio_loffset > HS_MAXFILEOFF || /* Offset too big. */
198 uiop->uio_loffset >= filesize) /* Past EOF. */
199 return (0);
200
201 do {
202 /*
203 * We want to ask for only the "right" amount of data.
204 * In this case that means:-
205 *
206 * We can't get data from beyond our EOF. If asked,
207 * we will give a short read.
208 *
209 * segmap_getmapflt returns buffers of MAXBSIZE bytes.
210 * These buffers are always MAXBSIZE aligned.
211 * If our starting offset is not MAXBSIZE aligned,
212 * we can only ask for less than MAXBSIZE bytes.
213 *
214 * If our requested offset and length are such that
215 * they belong in different MAXBSIZE aligned slots
216 * then we'll be making more than one call on
217 * segmap_getmapflt.
218 *
219 * This diagram shows the variables we use and their
220 * relationships.
221 *
222 * |<-----MAXBSIZE----->|
223 * +--------------------------...+
224 * |.....mapon->|<--n-->|....*...|EOF
225 * +--------------------------...+
226 * uio_loffset->|
227 * uio_resid....|<---------->|
228 * diff.........|<-------------->|
229 *
230 * So, in this case our offset is not aligned
231 * and our request takes us outside of the
232 * MAXBSIZE window. We will break this up into
233 * two segmap_getmapflt calls.
234 */
235 size_t nbytes;
236 offset_t mapon;
237 size_t n;
238 uint_t flags;
239
240 mapon = uiop->uio_loffset & MAXBOFFSET;
241 diff = filesize - uiop->uio_loffset;
242 nbytes = (size_t)MIN(MAXBSIZE - mapon, uiop->uio_resid);
243 n = MIN(diff, nbytes);
244 if (n <= 0) {
245 /* EOF or request satisfied. */
246 return (0);
247 }
248
249 /*
250 * Freebehind computation taken from:
251 * usr/src/uts/common/fs/ufs/ufs_vnops.c
252 */
253 if (drv_hztousec(ddi_get_lbolt()) >= smallfile_update) {
254 uint64_t percpufreeb;
255 if (smallfile1_d == 0) smallfile1_d = SMALLFILE1_D;
256 if (smallfile2_d == 0) smallfile2_d = SMALLFILE2_D;
257 percpufreeb = ptob((uint64_t)freemem) / ncpus_online;
258 smallfile1 = percpufreeb / smallfile1_d;
259 smallfile2 = percpufreeb / smallfile2_d;
260 smallfile1 = MAX(smallfile1, smallfile);
261 smallfile1 = MAX(smallfile1, smallfile64);
262 smallfile2 = MAX(smallfile1, smallfile2);
263 smallfile_update = drv_hztousec(ddi_get_lbolt())
264 + 1000000;
265 }
266
267 dofree = freebehind &&
268 hp->hs_prev_offset == uiop->uio_loffset &&
269 hp->hs_ra_bytes > 0;
270
271 base = segmap_getmapflt(segkmap, vp,
272 (u_offset_t)uiop->uio_loffset, n, 1, S_READ);
273
274 error = uiomove(base + mapon, n, UIO_READ, uiop);
275
276 if (error == 0) {
277 /*
278 * if read a whole block, or read to eof,
279 * won't need this buffer again soon.
280 */
281 if (n + mapon == MAXBSIZE ||
282 uiop->uio_loffset == filesize)
283 flags = SM_DONTNEED;
284 else
285 flags = 0;
286
287 if (dofree) {
288 flags = SM_FREE | SM_ASYNC;
289 if ((cache_read_ahead == 0) &&
290 uiop->uio_loffset > smallfile2)
291 flags |= SM_DONTNEED;
292 }
293
294 error = segmap_release(segkmap, base, flags);
295 } else
296 (void) segmap_release(segkmap, base, 0);
297 } while (error == 0 && uiop->uio_resid > 0);
298
299 return (error);
300 }
301
302 /*ARGSUSED2*/
303 static int
304 hsfs_getattr(
305 struct vnode *vp,
306 struct vattr *vap,
307 int flags,
308 struct cred *cred,
309 caller_context_t *ct)
310 {
311 struct hsnode *hp;
312 struct vfs *vfsp;
313 struct hsfs *fsp;
314
315 hp = VTOH(vp);
316 fsp = VFS_TO_HSFS(vp->v_vfsp);
317 vfsp = vp->v_vfsp;
318
319 if ((hp->hs_dirent.ext_size == 0) && (vp->v_type == VDIR)) {
320 hs_filldirent(vp, &hp->hs_dirent);
321 }
322 vap->va_type = IFTOVT(hp->hs_dirent.mode);
323 vap->va_mode = hp->hs_dirent.mode;
324 vap->va_uid = hp->hs_dirent.uid;
325 vap->va_gid = hp->hs_dirent.gid;
326
327 vap->va_fsid = vfsp->vfs_dev;
328 vap->va_nodeid = (ino64_t)hp->hs_nodeid;
329 vap->va_nlink = hp->hs_dirent.nlink;
330 vap->va_size = (offset_t)hp->hs_dirent.ext_size;
331
332 vap->va_atime.tv_sec = hp->hs_dirent.adate.tv_sec;
333 vap->va_atime.tv_nsec = hp->hs_dirent.adate.tv_usec*1000;
334 vap->va_mtime.tv_sec = hp->hs_dirent.mdate.tv_sec;
335 vap->va_mtime.tv_nsec = hp->hs_dirent.mdate.tv_usec*1000;
336 vap->va_ctime.tv_sec = hp->hs_dirent.cdate.tv_sec;
337 vap->va_ctime.tv_nsec = hp->hs_dirent.cdate.tv_usec*1000;
338 if (vp->v_type == VCHR || vp->v_type == VBLK)
339 vap->va_rdev = hp->hs_dirent.r_dev;
340 else
341 vap->va_rdev = 0;
342 vap->va_blksize = vfsp->vfs_bsize;
343 /* no. of blocks = no. of data blocks + no. of xar blocks */
344 vap->va_nblocks = (fsblkcnt64_t)howmany(vap->va_size + (u_longlong_t)
345 (hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift), DEV_BSIZE);
346 vap->va_seq = hp->hs_seq;
347 return (0);
348 }
349
350 /*ARGSUSED*/
351 static int
352 hsfs_readlink(struct vnode *vp,
353 struct uio *uiop,
354 struct cred *cred,
355 caller_context_t *ct)
356 {
357 struct hsnode *hp;
358
359 if (vp->v_type != VLNK)
360 return (EINVAL);
361
362 hp = VTOH(vp);
363
364 if (hp->hs_dirent.sym_link == (char *)NULL)
365 return (ENOENT);
366
367 return (uiomove(hp->hs_dirent.sym_link,
368 (size_t)MIN(hp->hs_dirent.ext_size,
369 uiop->uio_resid), UIO_READ, uiop));
370 }
371
372 /*ARGSUSED*/
373 static void
374 hsfs_inactive(struct vnode *vp,
375 struct cred *cred,
376 caller_context_t *ct)
377 {
378 struct hsnode *hp;
379 struct hsfs *fsp;
380
381 int nopage;
382
383 hp = VTOH(vp);
384 fsp = VFS_TO_HSFS(vp->v_vfsp);
385 /*
386 * Note: acquiring and holding v_lock for quite a while
387 * here serializes on the vnode; this is unfortunate, but
388 * likely not to overly impact performance, as the underlying
389 * device (CDROM drive) is quite slow.
390 */
391 rw_enter(&fsp->hsfs_hash_lock, RW_WRITER);
392 mutex_enter(&hp->hs_contents_lock);
393 mutex_enter(&vp->v_lock);
394
395 if (vp->v_count < 1) {
396 panic("hsfs_inactive: v_count < 1");
397 /*NOTREACHED*/
398 }
399
400 if (vp->v_count > 1 || (hp->hs_flags & HREF) == 0) {
401 vp->v_count--; /* release hold from vn_rele */
402 mutex_exit(&vp->v_lock);
403 mutex_exit(&hp->hs_contents_lock);
404 rw_exit(&fsp->hsfs_hash_lock);
405 return;
406 }
407 vp->v_count--; /* release hold from vn_rele */
408 if (vp->v_count == 0) {
409 /*
410 * Free the hsnode.
411 * If there are no pages associated with the
412 * hsnode, give it back to the kmem_cache,
413 * else put at the end of this file system's
414 * internal free list.
415 */
416 nopage = !vn_has_cached_data(vp);
417 hp->hs_flags = 0;
418 /*
419 * exit these locks now, since hs_freenode may
420 * kmem_free the hsnode and embedded vnode
421 */
422 mutex_exit(&vp->v_lock);
423 mutex_exit(&hp->hs_contents_lock);
424 hs_freenode(vp, fsp, nopage);
425 } else {
426 mutex_exit(&vp->v_lock);
427 mutex_exit(&hp->hs_contents_lock);
428 }
429 rw_exit(&fsp->hsfs_hash_lock);
430 }
431
432
433 /*ARGSUSED*/
434 static int
435 hsfs_lookup(
436 struct vnode *dvp,
437 char *nm,
438 struct vnode **vpp,
439 struct pathname *pnp,
440 int flags,
441 struct vnode *rdir,
442 struct cred *cred,
443 caller_context_t *ct,
444 int *direntflags,
445 pathname_t *realpnp)
446 {
447 int error;
448 int namelen = (int)strlen(nm);
449
450 if (*nm == '\0') {
451 VN_HOLD(dvp);
452 *vpp = dvp;
453 return (0);
454 }
455
456 /*
457 * If we're looking for ourself, life is simple.
458 */
459 if (namelen == 1 && *nm == '.') {
460 if (error = hs_access(dvp, (mode_t)VEXEC, cred))
461 return (error);
462 VN_HOLD(dvp);
463 *vpp = dvp;
464 return (0);
465 }
466
467 return (hs_dirlook(dvp, nm, namelen, vpp, cred));
468 }
469
470
471 /*ARGSUSED*/
472 static int
473 hsfs_readdir(
474 struct vnode *vp,
475 struct uio *uiop,
476 struct cred *cred,
477 int *eofp,
478 caller_context_t *ct,
479 int flags)
480 {
481 struct hsnode *dhp;
482 struct hsfs *fsp;
483 struct hs_direntry hd;
484 struct dirent64 *nd;
485 int error;
486 uint_t offset; /* real offset in directory */
487 uint_t dirsiz; /* real size of directory */
488 uchar_t *blkp;
489 int hdlen; /* length of hs directory entry */
490 long ndlen; /* length of dirent entry */
491 int bytes_wanted;
492 size_t bufsize; /* size of dirent buffer */
493 char *outbuf; /* ptr to dirent buffer */
494 char *dname;
495 int dnamelen;
496 size_t dname_size;
497 struct fbuf *fbp;
498 uint_t last_offset; /* last index into current dir block */
499 ino64_t dirino; /* temporary storage before storing in dirent */
500 off_t diroff;
501
502 dhp = VTOH(vp);
503 fsp = VFS_TO_HSFS(vp->v_vfsp);
504 if (dhp->hs_dirent.ext_size == 0)
505 hs_filldirent(vp, &dhp->hs_dirent);
506 dirsiz = dhp->hs_dirent.ext_size;
507 if (uiop->uio_loffset >= dirsiz) { /* at or beyond EOF */
508 if (eofp)
509 *eofp = 1;
510 return (0);
511 }
512 ASSERT(uiop->uio_loffset <= HS_MAXFILEOFF);
513 offset = uiop->uio_loffset;
514
515 dname_size = fsp->hsfs_namemax + 1; /* 1 for the ending NUL */
516 dname = kmem_alloc(dname_size, KM_SLEEP);
517 bufsize = uiop->uio_resid + sizeof (struct dirent64);
518
519 outbuf = kmem_alloc(bufsize, KM_SLEEP);
520 nd = (struct dirent64 *)outbuf;
521
522 while (offset < dirsiz) {
523 bytes_wanted = MIN(MAXBSIZE, dirsiz - (offset & MAXBMASK));
524
525 error = fbread(vp, (offset_t)(offset & MAXBMASK),
526 (unsigned int)bytes_wanted, S_READ, &fbp);
527 if (error)
528 goto done;
529
530 blkp = (uchar_t *)fbp->fb_addr;
531 last_offset = (offset & MAXBMASK) + fbp->fb_count;
532
533 #define rel_offset(offset) ((offset) & MAXBOFFSET) /* index into blkp */
534
535 while (offset < last_offset) {
536 /*
537 * Very similar validation code is found in
538 * process_dirblock(), hsfs_node.c.
539 * For an explanation, see there.
540 * It may make sense for the future to
541 * "consolidate" the code in hs_parsedir(),
542 * process_dirblock() and hsfs_readdir() into
543 * a single utility function.
544 */
545 hdlen = (int)((uchar_t)
546 HDE_DIR_LEN(&blkp[rel_offset(offset)]));
547 if (hdlen < HDE_ROOT_DIR_REC_SIZE ||
548 offset + hdlen > last_offset) {
549 /*
550 * advance to next sector boundary
551 */
552 offset = roundup(offset + 1, HS_SECTOR_SIZE);
553 if (hdlen)
554 hs_log_bogus_disk_warning(fsp,
555 HSFS_ERR_TRAILING_JUNK, 0);
556
557 continue;
558 }
559
560 bzero(&hd, sizeof (hd));
561
562 /*
563 * Just ignore invalid directory entries.
564 * XXX - maybe hs_parsedir() will detect EXISTENCE bit
565 */
566 if (!hs_parsedir(fsp, &blkp[rel_offset(offset)],
567 &hd, dname, &dnamelen, last_offset - offset)) {
568 /*
569 * Determine if there is enough room
570 */
571 ndlen = (long)DIRENT64_RECLEN((dnamelen));
572
573 if ((ndlen + ((char *)nd - outbuf)) >
574 uiop->uio_resid) {
575 fbrelse(fbp, S_READ);
576 goto done; /* output buffer full */
577 }
578
579 diroff = offset + hdlen;
580 /*
581 * If the media carries rrip-v1.12 or newer,
582 * and we trust the inodes from the rrip data
583 * (use_rrip_inodes != 0), use that data. If the
584 * media has been created by a recent mkisofs
585 * version, we may trust all numbers in the
586 * starting extent number; otherwise, we cannot
587 * do this for zero sized files and symlinks,
588 * because if we did we'd end up mapping all of
589 * them to the same node. We use HS_DUMMY_INO
590 * in this case and make sure that we will not
591 * map all files to the same meta data.
592 */
593 if (hd.inode != 0 && use_rrip_inodes) {
594 dirino = hd.inode;
595 } else if ((hd.ext_size == 0 ||
596 hd.sym_link != (char *)NULL) &&
597 (fsp->hsfs_flags & HSFSMNT_INODE) == 0) {
598 dirino = HS_DUMMY_INO;
599 } else {
600 dirino = hd.ext_lbn;
601 }
602
603 /* strncpy(9f) will zero uninitialized bytes */
604
605 ASSERT(strlen(dname) + 1 <=
606 DIRENT64_NAMELEN(ndlen));
607 (void) strncpy(nd->d_name, dname,
608 DIRENT64_NAMELEN(ndlen));
609 nd->d_reclen = (ushort_t)ndlen;
610 nd->d_off = (offset_t)diroff;
611 nd->d_ino = dirino;
612 nd = (struct dirent64 *)((char *)nd + ndlen);
613
614 /*
615 * free up space allocated for symlink
616 */
617 if (hd.sym_link != (char *)NULL) {
618 kmem_free(hd.sym_link,
619 (size_t)(hd.ext_size+1));
620 hd.sym_link = (char *)NULL;
621 }
622 }
623 offset += hdlen;
624 }
625 fbrelse(fbp, S_READ);
626 }
627
628 /*
629 * Got here for one of the following reasons:
630 * 1) outbuf is full (error == 0)
631 * 2) end of directory reached (error == 0)
632 * 3) error reading directory sector (error != 0)
633 * 4) directory entry crosses sector boundary (error == 0)
634 *
635 * If any directory entries have been copied, don't report
636 * case 4. Instead, return the valid directory entries.
637 *
638 * If no entries have been copied, report the error.
639 * If case 4, this will be indistiguishable from EOF.
640 */
641 done:
642 ndlen = ((char *)nd - outbuf);
643 if (ndlen != 0) {
644 error = uiomove(outbuf, (size_t)ndlen, UIO_READ, uiop);
645 uiop->uio_loffset = offset;
646 }
647 kmem_free(dname, dname_size);
648 kmem_free(outbuf, bufsize);
649 if (eofp && error == 0)
650 *eofp = (uiop->uio_loffset >= dirsiz);
651 return (error);
652 }
653
654 /*ARGSUSED2*/
655 static int
656 hsfs_fid(struct vnode *vp, struct fid *fidp, caller_context_t *ct)
657 {
658 struct hsnode *hp;
659 struct hsfid *fid;
660
661 if (fidp->fid_len < (sizeof (*fid) - sizeof (fid->hf_len))) {
662 fidp->fid_len = sizeof (*fid) - sizeof (fid->hf_len);
663 return (ENOSPC);
664 }
665
666 fid = (struct hsfid *)fidp;
667 fid->hf_len = sizeof (*fid) - sizeof (fid->hf_len);
668 hp = VTOH(vp);
669 mutex_enter(&hp->hs_contents_lock);
670 fid->hf_dir_lbn = hp->hs_dir_lbn;
671 fid->hf_dir_off = (ushort_t)hp->hs_dir_off;
672 fid->hf_ino = hp->hs_nodeid;
673 mutex_exit(&hp->hs_contents_lock);
674 return (0);
675 }
676
677 /*ARGSUSED*/
678 static int
679 hsfs_open(struct vnode **vpp,
680 int flag,
681 struct cred *cred,
682 caller_context_t *ct)
683 {
684 return (0);
685 }
686
687 /*ARGSUSED*/
688 static int
689 hsfs_close(
690 struct vnode *vp,
691 int flag,
692 int count,
693 offset_t offset,
694 struct cred *cred,
695 caller_context_t *ct)
696 {
697 (void) cleanlocks(vp, ttoproc(curthread)->p_pid, 0);
698 cleanshares(vp, ttoproc(curthread)->p_pid);
699 return (0);
700 }
701
702 /*ARGSUSED2*/
703 static int
704 hsfs_access(struct vnode *vp,
705 int mode,
706 int flags,
707 cred_t *cred,
708 caller_context_t *ct)
709 {
710 return (hs_access(vp, (mode_t)mode, cred));
711 }
712
713 /*
714 * the seek time of a CD-ROM is very slow, and data transfer
715 * rate is even worse (max. 150K per sec). The design
716 * decision is to reduce access to cd-rom as much as possible,
717 * and to transfer a sizable block (read-ahead) of data at a time.
718 * UFS style of read ahead one block at a time is not appropriate,
719 * and is not supported
720 */
721
722 /*
723 * KLUSTSIZE should be a multiple of PAGESIZE and <= MAXPHYS.
724 */
725 #define KLUSTSIZE (56 * 1024)
726 /* we don't support read ahead */
727 int hsfs_lostpage; /* no. of times we lost original page */
728
729 /*
730 * Used to prevent biodone() from releasing buf resources that
731 * we didn't allocate in quite the usual way.
732 */
733 /*ARGSUSED*/
734 int
735 hsfs_iodone(struct buf *bp)
736 {
737 sema_v(&bp->b_io);
738 return (0);
739 }
740
741 /*
742 * The taskq thread that invokes the scheduling function to ensure
743 * that all readaheads are complete and cleans up the associated
744 * memory and releases the page lock.
745 */
746 void
747 hsfs_ra_task(void *arg)
748 {
749 struct hio_info *info = arg;
750 uint_t count;
751 struct buf *wbuf;
752
753 ASSERT(info->pp != NULL);
754
755 for (count = 0; count < info->bufsused; count++) {
756 wbuf = &(info->bufs[count]);
757
758 DTRACE_PROBE1(hsfs_io_wait_ra, struct buf *, wbuf);
759 while (sema_tryp(&(info->sema[count])) == 0) {
760 if (hsched_invoke_strategy(info->fsp)) {
761 sema_p(&(info->sema[count]));
762 break;
763 }
764 }
765 sema_destroy(&(info->sema[count]));
766 DTRACE_PROBE1(hsfs_io_done_ra, struct buf *, wbuf);
767 biofini(&(info->bufs[count]));
768 }
769 for (count = 0; count < info->bufsused; count++) {
770 if (info->vas[count] != NULL) {
771 ppmapout(info->vas[count]);
772 }
773 }
774 kmem_free(info->vas, info->bufcnt * sizeof (caddr_t));
775 kmem_free(info->bufs, info->bufcnt * sizeof (struct buf));
776 kmem_free(info->sema, info->bufcnt * sizeof (ksema_t));
777
778 pvn_read_done(info->pp, 0);
779 kmem_cache_free(hio_info_cache, info);
780 }
781
782 /*
783 * Submit asynchronous readahead requests to the I/O scheduler
784 * depending on the number of pages to read ahead. These requests
785 * are asynchronous to the calling thread but I/O requests issued
786 * subsequently by other threads with higher LBNs must wait for
787 * these readaheads to complete since we have a single ordered
788 * I/O pipeline. Thus these readaheads are semi-asynchronous.
789 * A TaskQ handles waiting for the readaheads to complete.
790 *
791 * This function is mostly a copy of hsfs_getapage but somewhat
792 * simpler. A readahead request is aborted if page allocation
793 * fails.
794 */
795 /*ARGSUSED*/
796 static int
797 hsfs_getpage_ra(
798 struct vnode *vp,
799 u_offset_t off,
800 struct seg *seg,
801 caddr_t addr,
802 struct hsnode *hp,
803 struct hsfs *fsp,
804 int xarsiz,
805 offset_t bof,
806 int chunk_lbn_count,
807 int chunk_data_bytes)
808 {
809 struct buf *bufs;
810 caddr_t *vas;
811 caddr_t va;
812 struct page *pp, *searchp, *lastp;
813 struct vnode *devvp;
814 ulong_t byte_offset;
815 size_t io_len_tmp;
816 uint_t io_off, io_len;
817 uint_t xlen;
818 uint_t filsiz;
819 uint_t secsize;
820 uint_t bufcnt;
821 uint_t bufsused;
822 uint_t count;
823 uint_t io_end;
824 uint_t which_chunk_lbn;
825 uint_t offset_lbn;
826 uint_t offset_extra;
827 offset_t offset_bytes;
828 uint_t remaining_bytes;
829 uint_t extension;
830 int remainder; /* must be signed */
831 diskaddr_t driver_block;
832 u_offset_t io_off_tmp;
833 ksema_t *fio_done;
834 struct hio_info *info;
835 size_t len;
836
837 ASSERT(fsp->hqueue != NULL);
838
839 if (addr >= seg->s_base + seg->s_size) {
840 return (-1);
841 }
842
843 devvp = fsp->hsfs_devvp;
844 secsize = fsp->hsfs_vol.lbn_size; /* bytes per logical block */
845
846 /* file data size */
847 filsiz = hp->hs_dirent.ext_size;
848
849 if (off >= filsiz)
850 return (0);
851
852 extension = 0;
853 pp = NULL;
854
855 extension += hp->hs_ra_bytes;
856
857 /*
858 * Some CD writers (e.g. Kodak Photo CD writers)
859 * create CDs in TAO mode and reserve tracks that
860 * are not completely written. Some sectors remain
861 * unreadable for this reason and give I/O errors.
862 * Also, there's no point in reading sectors
863 * we'll never look at. So, if we're asked to go
864 * beyond the end of a file, truncate to the length
865 * of that file.
866 *
867 * Additionally, this behaviour is required by section
868 * 6.4.5 of ISO 9660:1988(E).
869 */
870 len = MIN(extension ? extension : PAGESIZE, filsiz - off);
871
872 /* A little paranoia */
873 if (len <= 0)
874 return (-1);
875
876 /*
877 * After all that, make sure we're asking for things in units
878 * that bdev_strategy() will understand (see bug 4202551).
879 */
880 len = roundup(len, DEV_BSIZE);
881
882 pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
883 &io_len_tmp, off, len, 1);
884
885 if (pp == NULL) {
886 hp->hs_num_contig = 0;
887 hp->hs_ra_bytes = 0;
888 hp->hs_prev_offset = 0;
889 return (-1);
890 }
891
892 io_off = (uint_t)io_off_tmp;
893 io_len = (uint_t)io_len_tmp;
894
895 /* check for truncation */
896 /*
897 * xxx Clean up and return EIO instead?
898 * xxx Ought to go to u_offset_t for everything, but we
899 * xxx call lots of things that want uint_t arguments.
900 */
901 ASSERT(io_off == io_off_tmp);
902
903 /*
904 * get enough buffers for worst-case scenario
905 * (i.e., no coalescing possible).
906 */
907 bufcnt = (len + secsize - 1) / secsize;
908 bufs = kmem_alloc(bufcnt * sizeof (struct buf), KM_SLEEP);
909 vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
910
911 /*
912 * Allocate a array of semaphores since we are doing I/O
913 * scheduling.
914 */
915 fio_done = kmem_alloc(bufcnt * sizeof (ksema_t), KM_SLEEP);
916
917 /*
918 * If our filesize is not an integer multiple of PAGESIZE,
919 * we zero that part of the last page that's between EOF and
920 * the PAGESIZE boundary.
921 */
922 xlen = io_len & PAGEOFFSET;
923 if (xlen != 0)
924 pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
925
926 DTRACE_PROBE2(hsfs_readahead, struct vnode *, vp, uint_t, io_len);
927
928 va = NULL;
929 lastp = NULL;
930 searchp = pp;
931 io_end = io_off + io_len;
932 for (count = 0, byte_offset = io_off;
933 byte_offset < io_end;
934 count++) {
935 ASSERT(count < bufcnt);
936
937 bioinit(&bufs[count]);
938 bufs[count].b_edev = devvp->v_rdev;
939 bufs[count].b_dev = cmpdev(devvp->v_rdev);
940 bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
941 bufs[count].b_iodone = hsfs_iodone;
942 bufs[count].b_vp = vp;
943 bufs[count].b_file = vp;
944
945 /* Compute disk address for interleaving. */
946
947 /* considered without skips */
948 which_chunk_lbn = byte_offset / chunk_data_bytes;
949
950 /* factor in skips */
951 offset_lbn = which_chunk_lbn * chunk_lbn_count;
952
953 /* convert to physical byte offset for lbn */
954 offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
955
956 /* don't forget offset into lbn */
957 offset_extra = byte_offset % chunk_data_bytes;
958
959 /* get virtual block number for driver */
960 driver_block = lbtodb(bof + xarsiz
961 + offset_bytes + offset_extra);
962
963 if (lastp != searchp) {
964 /* this branch taken first time through loop */
965 va = vas[count] = ppmapin(searchp, PROT_WRITE,
966 (caddr_t)-1);
967 /* ppmapin() guarantees not to return NULL */
968 } else {
969 vas[count] = NULL;
970 }
971
972 bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
973 bufs[count].b_offset =
974 (offset_t)(byte_offset - io_off + off);
975
976 /*
977 * We specifically use the b_lblkno member here
978 * as even in the 32 bit world driver_block can
979 * get very large in line with the ISO9660 spec.
980 */
981
982 bufs[count].b_lblkno = driver_block;
983
984 remaining_bytes = ((which_chunk_lbn + 1) * chunk_data_bytes)
985 - byte_offset;
986
987 /*
988 * remaining_bytes can't be zero, as we derived
989 * which_chunk_lbn directly from byte_offset.
990 */
991 if ((remaining_bytes + byte_offset) < (off + len)) {
992 /* coalesce-read the rest of the chunk */
993 bufs[count].b_bcount = remaining_bytes;
994 } else {
995 /* get the final bits */
996 bufs[count].b_bcount = off + len - byte_offset;
997 }
998
999 remainder = PAGESIZE - (byte_offset % PAGESIZE);
1000 if (bufs[count].b_bcount > remainder) {
1001 bufs[count].b_bcount = remainder;
1002 }
1003
1004 bufs[count].b_bufsize = bufs[count].b_bcount;
1005 if (((offset_t)byte_offset + bufs[count].b_bcount) >
1006 HS_MAXFILEOFF) {
1007 break;
1008 }
1009 byte_offset += bufs[count].b_bcount;
1010
1011 /*
1012 * We are scheduling I/O so we need to enqueue
1013 * requests rather than calling bdev_strategy
1014 * here. A later invocation of the scheduling
1015 * function will take care of doing the actual
1016 * I/O as it selects requests from the queue as
1017 * per the scheduling logic.
1018 */
1019 struct hio *hsio = kmem_cache_alloc(hio_cache,
1020 KM_SLEEP);
1021
1022 sema_init(&fio_done[count], 0, NULL,
1023 SEMA_DEFAULT, NULL);
1024 hsio->bp = &bufs[count];
1025 hsio->sema = &fio_done[count];
1026 hsio->io_lblkno = bufs[count].b_lblkno;
1027 hsio->nblocks = howmany(hsio->bp->b_bcount,
1028 DEV_BSIZE);
1029
1030 /* used for deadline */
1031 hsio->io_timestamp = drv_hztousec(ddi_get_lbolt());
1032
1033 /* for I/O coalescing */
1034 hsio->contig_chain = NULL;
1035 hsched_enqueue_io(fsp, hsio, 1);
1036
1037 lwp_stat_update(LWP_STAT_INBLK, 1);
1038 lastp = searchp;
1039 if ((remainder - bufs[count].b_bcount) < 1) {
1040 searchp = searchp->p_next;
1041 }
1042 }
1043
1044 bufsused = count;
1045 info = kmem_cache_alloc(hio_info_cache, KM_SLEEP);
1046 info->bufs = bufs;
1047 info->vas = vas;
1048 info->sema = fio_done;
1049 info->bufsused = bufsused;
1050 info->bufcnt = bufcnt;
1051 info->fsp = fsp;
1052 info->pp = pp;
1053
1054 (void) taskq_dispatch(fsp->hqueue->ra_task,
1055 hsfs_ra_task, info, KM_SLEEP);
1056 /*
1057 * The I/O locked pages are unlocked in our taskq thread.
1058 */
1059 return (0);
1060 }
1061
1062 /*
1063 * Each file may have a different interleaving on disk. This makes
1064 * things somewhat interesting. The gist is that there are some
1065 * number of contiguous data sectors, followed by some other number
1066 * of contiguous skip sectors. The sum of those two sets of sectors
1067 * defines the interleave size. Unfortunately, it means that we generally
1068 * can't simply read N sectors starting at a given offset to satisfy
1069 * any given request.
1070 *
1071 * What we do is get the relevant memory pages via pvn_read_kluster(),
1072 * then stride through the interleaves, setting up a buf for each
1073 * sector that needs to be brought in. Instead of kmem_alloc'ing
1074 * space for the sectors, though, we just point at the appropriate
1075 * spot in the relevant page for each of them. This saves us a bunch
1076 * of copying.
1077 *
1078 * NOTICE: The code below in hsfs_getapage is mostly same as the code
1079 * in hsfs_getpage_ra above (with some omissions). If you are
1080 * making any change to this function, please also look at
1081 * hsfs_getpage_ra.
1082 */
1083 /*ARGSUSED*/
1084 static int
1085 hsfs_getapage(
1086 struct vnode *vp,
1087 u_offset_t off,
1088 size_t len,
1089 uint_t *protp,
1090 struct page *pl[],
1091 size_t plsz,
1092 struct seg *seg,
1093 caddr_t addr,
1094 enum seg_rw rw,
1095 struct cred *cred)
1096 {
1097 struct hsnode *hp;
1098 struct hsfs *fsp;
1099 int err;
1100 struct buf *bufs;
1101 caddr_t *vas;
1102 caddr_t va;
1103 struct page *pp, *searchp, *lastp;
1104 page_t *pagefound;
1105 offset_t bof;
1106 struct vnode *devvp;
1107 ulong_t byte_offset;
1108 size_t io_len_tmp;
1109 uint_t io_off, io_len;
1110 uint_t xlen;
1111 uint_t filsiz;
1112 uint_t secsize;
1113 uint_t bufcnt;
1114 uint_t bufsused;
1115 uint_t count;
1116 uint_t io_end;
1117 uint_t which_chunk_lbn;
1118 uint_t offset_lbn;
1119 uint_t offset_extra;
1120 offset_t offset_bytes;
1121 uint_t remaining_bytes;
1122 uint_t extension;
1123 int remainder; /* must be signed */
1124 int chunk_lbn_count;
1125 int chunk_data_bytes;
1126 int xarsiz;
1127 diskaddr_t driver_block;
1128 u_offset_t io_off_tmp;
1129 ksema_t *fio_done;
1130 int calcdone;
1131
1132 /*
1133 * We don't support asynchronous operation at the moment, so
1134 * just pretend we did it. If the pages are ever actually
1135 * needed, they'll get brought in then.
1136 */
1137 if (pl == NULL)
1138 return (0);
1139
1140 hp = VTOH(vp);
1141 fsp = VFS_TO_HSFS(vp->v_vfsp);
1142 devvp = fsp->hsfs_devvp;
1143 secsize = fsp->hsfs_vol.lbn_size; /* bytes per logical block */
1144
1145 /* file data size */
1146 filsiz = hp->hs_dirent.ext_size;
1147
1148 /* disk addr for start of file */
1149 bof = LBN_TO_BYTE((offset_t)hp->hs_dirent.ext_lbn, vp->v_vfsp);
1150
1151 /* xarsiz byte must be skipped for data */
1152 xarsiz = hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift;
1153
1154 /* how many logical blocks in an interleave (data+skip) */
1155 chunk_lbn_count = hp->hs_dirent.intlf_sz + hp->hs_dirent.intlf_sk;
1156
1157 if (chunk_lbn_count == 0) {
1158 chunk_lbn_count = 1;
1159 }
1160
1161 /*
1162 * Convert interleaving size into bytes. The zero case
1163 * (no interleaving) optimization is handled as a side-
1164 * effect of the read-ahead logic.
1165 */
1166 if (hp->hs_dirent.intlf_sz == 0) {
1167 chunk_data_bytes = LBN_TO_BYTE(1, vp->v_vfsp);
1168 /*
1169 * Optimization: If our pagesize is a multiple of LBN
1170 * bytes, we can avoid breaking up a page into individual
1171 * lbn-sized requests.
1172 */
1173 if (PAGESIZE % chunk_data_bytes == 0) {
1174 chunk_lbn_count = BYTE_TO_LBN(PAGESIZE, vp->v_vfsp);
1175 chunk_data_bytes = PAGESIZE;
1176 }
1177 } else {
1178 chunk_data_bytes =
1179 LBN_TO_BYTE(hp->hs_dirent.intlf_sz, vp->v_vfsp);
1180 }
1181
1182 reread:
1183 err = 0;
1184 pagefound = 0;
1185 calcdone = 0;
1186
1187 /*
1188 * Do some read-ahead. This mostly saves us a bit of
1189 * system cpu time more than anything else when doing
1190 * sequential reads. At some point, could do the
1191 * read-ahead asynchronously which might gain us something
1192 * on wall time, but it seems unlikely....
1193 *
1194 * We do the easy case here, which is to read through
1195 * the end of the chunk, minus whatever's at the end that
1196 * won't exactly fill a page.
1197 */
1198 if (hp->hs_ra_bytes > 0 && chunk_data_bytes != PAGESIZE) {
1199 which_chunk_lbn = (off + len) / chunk_data_bytes;
1200 extension = ((which_chunk_lbn + 1) * chunk_data_bytes) - off;
1201 extension -= (extension % PAGESIZE);
1202 } else {
1203 extension = roundup(len, PAGESIZE);
1204 }
1205
1206 atomic_inc_64(&fsp->total_pages_requested);
1207
1208 pp = NULL;
1209 again:
1210 /* search for page in buffer */
1211 if ((pagefound = page_exists(vp, off)) == 0) {
1212 /*
1213 * Need to really do disk IO to get the page.
1214 */
1215 if (!calcdone) {
1216 extension += hp->hs_ra_bytes;
1217
1218 /*
1219 * Some cd writers don't write sectors that aren't
1220 * used. Also, there's no point in reading sectors
1221 * we'll never look at. So, if we're asked to go
1222 * beyond the end of a file, truncate to the length
1223 * of that file.
1224 *
1225 * Additionally, this behaviour is required by section
1226 * 6.4.5 of ISO 9660:1988(E).
1227 */
1228 len = MIN(extension ? extension : PAGESIZE,
1229 filsiz - off);
1230
1231 /* A little paranoia. */
1232 ASSERT(len > 0);
1233
1234 /*
1235 * After all that, make sure we're asking for things
1236 * in units that bdev_strategy() will understand
1237 * (see bug 4202551).
1238 */
1239 len = roundup(len, DEV_BSIZE);
1240 calcdone = 1;
1241 }
1242
1243 pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
1244 &io_len_tmp, off, len, 0);
1245
1246 if (pp == NULL) {
1247 /*
1248 * Pressure on memory, roll back readahead
1249 */
1250 hp->hs_num_contig = 0;
1251 hp->hs_ra_bytes = 0;
1252 hp->hs_prev_offset = 0;
1253 goto again;
1254 }
1255
1256 io_off = (uint_t)io_off_tmp;
1257 io_len = (uint_t)io_len_tmp;
1258
1259 /* check for truncation */
1260 /*
1261 * xxx Clean up and return EIO instead?
1262 * xxx Ought to go to u_offset_t for everything, but we
1263 * xxx call lots of things that want uint_t arguments.
1264 */
1265 ASSERT(io_off == io_off_tmp);
1266
1267 /*
1268 * get enough buffers for worst-case scenario
1269 * (i.e., no coalescing possible).
1270 */
1271 bufcnt = (len + secsize - 1) / secsize;
1272 bufs = kmem_zalloc(bufcnt * sizeof (struct buf), KM_SLEEP);
1273 vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
1274
1275 /*
1276 * Allocate a array of semaphores if we are doing I/O
1277 * scheduling.
1278 */
1279 if (fsp->hqueue != NULL)
1280 fio_done = kmem_alloc(bufcnt * sizeof (ksema_t),
1281 KM_SLEEP);
1282 for (count = 0; count < bufcnt; count++) {
1283 bioinit(&bufs[count]);
1284 bufs[count].b_edev = devvp->v_rdev;
1285 bufs[count].b_dev = cmpdev(devvp->v_rdev);
1286 bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
1287 bufs[count].b_iodone = hsfs_iodone;
1288 bufs[count].b_vp = vp;
1289 bufs[count].b_file = vp;
1290 }
1291
1292 /*
1293 * If our filesize is not an integer multiple of PAGESIZE,
1294 * we zero that part of the last page that's between EOF and
1295 * the PAGESIZE boundary.
1296 */
1297 xlen = io_len & PAGEOFFSET;
1298 if (xlen != 0)
1299 pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
1300
1301 va = NULL;
1302 lastp = NULL;
1303 searchp = pp;
1304 io_end = io_off + io_len;
1305 for (count = 0, byte_offset = io_off;
1306 byte_offset < io_end; count++) {
1307 ASSERT(count < bufcnt);
1308
1309 /* Compute disk address for interleaving. */
1310
1311 /* considered without skips */
1312 which_chunk_lbn = byte_offset / chunk_data_bytes;
1313
1314 /* factor in skips */
1315 offset_lbn = which_chunk_lbn * chunk_lbn_count;
1316
1317 /* convert to physical byte offset for lbn */
1318 offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
1319
1320 /* don't forget offset into lbn */
1321 offset_extra = byte_offset % chunk_data_bytes;
1322
1323 /* get virtual block number for driver */
1324 driver_block =
1325 lbtodb(bof + xarsiz + offset_bytes + offset_extra);
1326
1327 if (lastp != searchp) {
1328 /* this branch taken first time through loop */
1329 va = vas[count] =
1330 ppmapin(searchp, PROT_WRITE, (caddr_t)-1);
1331 /* ppmapin() guarantees not to return NULL */
1332 } else {
1333 vas[count] = NULL;
1334 }
1335
1336 bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
1337 bufs[count].b_offset =
1338 (offset_t)(byte_offset - io_off + off);
1339
1340 /*
1341 * We specifically use the b_lblkno member here
1342 * as even in the 32 bit world driver_block can
1343 * get very large in line with the ISO9660 spec.
1344 */
1345
1346 bufs[count].b_lblkno = driver_block;
1347
1348 remaining_bytes =
1349 ((which_chunk_lbn + 1) * chunk_data_bytes)
1350 - byte_offset;
1351
1352 /*
1353 * remaining_bytes can't be zero, as we derived
1354 * which_chunk_lbn directly from byte_offset.
1355 */
1356 if ((remaining_bytes + byte_offset) < (off + len)) {
1357 /* coalesce-read the rest of the chunk */
1358 bufs[count].b_bcount = remaining_bytes;
1359 } else {
1360 /* get the final bits */
1361 bufs[count].b_bcount = off + len - byte_offset;
1362 }
1363
1364 /*
1365 * It would be nice to do multiple pages'
1366 * worth at once here when the opportunity
1367 * arises, as that has been shown to improve
1368 * our wall time. However, to do that
1369 * requires that we use the pageio subsystem,
1370 * which doesn't mix well with what we're
1371 * already using here. We can't use pageio
1372 * all the time, because that subsystem
1373 * assumes that a page is stored in N
1374 * contiguous blocks on the device.
1375 * Interleaving violates that assumption.
1376 *
1377 * Update: This is now not so big a problem
1378 * because of the I/O scheduler sitting below
1379 * that can re-order and coalesce I/O requests.
1380 */
1381
1382 remainder = PAGESIZE - (byte_offset % PAGESIZE);
1383 if (bufs[count].b_bcount > remainder) {
1384 bufs[count].b_bcount = remainder;
1385 }
1386
1387 bufs[count].b_bufsize = bufs[count].b_bcount;
1388 if (((offset_t)byte_offset + bufs[count].b_bcount) >
1389 HS_MAXFILEOFF) {
1390 break;
1391 }
1392 byte_offset += bufs[count].b_bcount;
1393
1394 if (fsp->hqueue == NULL) {
1395 (void) bdev_strategy(&bufs[count]);
1396
1397 } else {
1398 /*
1399 * We are scheduling I/O so we need to enqueue
1400 * requests rather than calling bdev_strategy
1401 * here. A later invocation of the scheduling
1402 * function will take care of doing the actual
1403 * I/O as it selects requests from the queue as
1404 * per the scheduling logic.
1405 */
1406 struct hio *hsio = kmem_cache_alloc(hio_cache,
1407 KM_SLEEP);
1408
1409 sema_init(&fio_done[count], 0, NULL,
1410 SEMA_DEFAULT, NULL);
1411 hsio->bp = &bufs[count];
1412 hsio->sema = &fio_done[count];
1413 hsio->io_lblkno = bufs[count].b_lblkno;
1414 hsio->nblocks = howmany(hsio->bp->b_bcount,
1415 DEV_BSIZE);
1416
1417 /* used for deadline */
1418 hsio->io_timestamp =
1419 drv_hztousec(ddi_get_lbolt());
1420
1421 /* for I/O coalescing */
1422 hsio->contig_chain = NULL;
1423 hsched_enqueue_io(fsp, hsio, 0);
1424 }
1425
1426 lwp_stat_update(LWP_STAT_INBLK, 1);
1427 lastp = searchp;
1428 if ((remainder - bufs[count].b_bcount) < 1) {
1429 searchp = searchp->p_next;
1430 }
1431 }
1432
1433 bufsused = count;
1434 /* Now wait for everything to come in */
1435 if (fsp->hqueue == NULL) {
1436 for (count = 0; count < bufsused; count++) {
1437 if (err == 0) {
1438 err = biowait(&bufs[count]);
1439 } else
1440 (void) biowait(&bufs[count]);
1441 }
1442 } else {
1443 for (count = 0; count < bufsused; count++) {
1444 struct buf *wbuf;
1445
1446 /*
1447 * Invoke scheduling function till our buf
1448 * is processed. In doing this it might
1449 * process bufs enqueued by other threads
1450 * which is good.
1451 */
1452 wbuf = &bufs[count];
1453 DTRACE_PROBE1(hsfs_io_wait, struct buf *, wbuf);
1454 while (sema_tryp(&fio_done[count]) == 0) {
1455 /*
1456 * hsched_invoke_strategy will return 1
1457 * if the I/O queue is empty. This means
1458 * that there is another thread who has
1459 * issued our buf and is waiting. So we
1460 * just block instead of spinning.
1461 */
1462 if (hsched_invoke_strategy(fsp)) {
1463 sema_p(&fio_done[count]);
1464 break;
1465 }
1466 }
1467 sema_destroy(&fio_done[count]);
1468 DTRACE_PROBE1(hsfs_io_done, struct buf *, wbuf);
1469
1470 if (err == 0) {
1471 err = geterror(wbuf);
1472 }
1473 }
1474 kmem_free(fio_done, bufcnt * sizeof (ksema_t));
1475 }
1476
1477 /* Don't leak resources */
1478 for (count = 0; count < bufcnt; count++) {
1479 biofini(&bufs[count]);
1480 if (count < bufsused && vas[count] != NULL) {
1481 ppmapout(vas[count]);
1482 }
1483 }
1484
1485 kmem_free(vas, bufcnt * sizeof (caddr_t));
1486 kmem_free(bufs, bufcnt * sizeof (struct buf));
1487 }
1488
1489 if (err) {
1490 pvn_read_done(pp, B_ERROR);
1491 return (err);
1492 }
1493
1494 /*
1495 * Lock the requested page, and the one after it if possible.
1496 * Don't bother if our caller hasn't given us a place to stash
1497 * the page pointers, since otherwise we'd lock pages that would
1498 * never get unlocked.
1499 */
1500 if (pagefound) {
1501 int index;
1502 ulong_t soff;
1503
1504 /*
1505 * Make sure it's in memory before we say it's here.
1506 */
1507 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) {
1508 hsfs_lostpage++;
1509 goto reread;
1510 }
1511
1512 pl[0] = pp;
1513 index = 1;
1514 atomic_inc_64(&fsp->cache_read_pages);
1515
1516 /*
1517 * Try to lock the next page, if it exists, without
1518 * blocking.
1519 */
1520 plsz -= PAGESIZE;
1521 /* LINTED (plsz is unsigned) */
1522 for (soff = off + PAGESIZE; plsz > 0;
1523 soff += PAGESIZE, plsz -= PAGESIZE) {
1524 pp = page_lookup_nowait(vp, (u_offset_t)soff,
1525 SE_SHARED);
1526 if (pp == NULL)
1527 break;
1528 pl[index++] = pp;
1529 }
1530 pl[index] = NULL;
1531
1532 /*
1533 * Schedule a semi-asynchronous readahead if we are
1534 * accessing the last cached page for the current
1535 * file.
1536 *
1537 * Doing this here means that readaheads will be
1538 * issued only if cache-hits occur. This is an advantage
1539 * since cache-hits would mean that readahead is giving
1540 * the desired benefit. If cache-hits do not occur there
1541 * is no point in reading ahead of time - the system
1542 * is loaded anyway.
1543 */
1544 if (fsp->hqueue != NULL &&
1545 hp->hs_prev_offset - off == PAGESIZE &&
1546 hp->hs_prev_offset < filsiz &&
1547 hp->hs_ra_bytes > 0 &&
1548 !page_exists(vp, hp->hs_prev_offset)) {
1549 (void) hsfs_getpage_ra(vp, hp->hs_prev_offset, seg,
1550 addr + PAGESIZE, hp, fsp, xarsiz, bof,
1551 chunk_lbn_count, chunk_data_bytes);
1552 }
1553
1554 return (0);
1555 }
1556
1557 if (pp != NULL) {
1558 pvn_plist_init(pp, pl, plsz, off, io_len, rw);
1559 }
1560
1561 return (err);
1562 }
1563
1564 /*ARGSUSED*/
1565 static int
1566 hsfs_getpage(
1567 struct vnode *vp,
1568 offset_t off,
1569 size_t len,
1570 uint_t *protp,
1571 struct page *pl[],
1572 size_t plsz,
1573 struct seg *seg,
1574 caddr_t addr,
1575 enum seg_rw rw,
1576 struct cred *cred,
1577 caller_context_t *ct)
1578 {
1579 uint_t filsiz;
1580 struct hsfs *fsp;
1581 struct hsnode *hp;
1582
1583 fsp = VFS_TO_HSFS(vp->v_vfsp);
1584 hp = VTOH(vp);
1585
1586 /* does not support write */
1587 if (rw == S_WRITE) {
1588 return (EROFS);
1589 }
1590
1591 if (vp->v_flag & VNOMAP) {
1592 return (ENOSYS);
1593 }
1594
1595 ASSERT(off <= HS_MAXFILEOFF);
1596
1597 /*
1598 * Determine file data size for EOF check.
1599 */
1600 filsiz = hp->hs_dirent.ext_size;
1601 if ((off + len) > (offset_t)(filsiz + PAGEOFFSET) && seg != segkmap)
1602 return (EFAULT); /* beyond EOF */
1603
1604 /*
1605 * Async Read-ahead computation.
1606 * This attempts to detect sequential access pattern and
1607 * enables reading extra pages ahead of time.
1608 */
1609 if (fsp->hqueue != NULL) {
1610 /*
1611 * This check for sequential access also takes into
1612 * account segmap weirdness when reading in chunks
1613 * less than the segmap size of 8K.
1614 */
1615 if (hp->hs_prev_offset == off || (off <
1616 hp->hs_prev_offset && off + MAX(len, PAGESIZE)
1617 >= hp->hs_prev_offset)) {
1618 if (hp->hs_num_contig <
1619 (seq_contig_requests - 1)) {
1620 hp->hs_num_contig++;
1621
1622 } else {
1623 /*
1624 * We increase readahead quantum till
1625 * a predefined max. max_readahead_bytes
1626 * is a multiple of PAGESIZE.
1627 */
1628 if (hp->hs_ra_bytes <
1629 fsp->hqueue->max_ra_bytes) {
1630 hp->hs_ra_bytes += PAGESIZE;
1631 }
1632 }
1633 } else {
1634 /*
1635 * Not contiguous so reduce read ahead counters.
1636 */
1637 if (hp->hs_ra_bytes > 0)
1638 hp->hs_ra_bytes -= PAGESIZE;
1639
1640 if (hp->hs_ra_bytes <= 0) {
1641 hp->hs_ra_bytes = 0;
1642 if (hp->hs_num_contig > 0)
1643 hp->hs_num_contig--;
1644 }
1645 }
1646 /*
1647 * Length must be rounded up to page boundary.
1648 * since we read in units of pages.
1649 */
1650 hp->hs_prev_offset = off + roundup(len, PAGESIZE);
1651 DTRACE_PROBE1(hsfs_compute_ra, struct hsnode *, hp);
1652 }
1653 if (protp != NULL)
1654 *protp = PROT_ALL;
1655
1656 return (pvn_getpages(hsfs_getapage, vp, off, len, protp, pl, plsz,
1657 seg, addr, rw, cred));
1658 }
1659
1660
1661
1662 /*
1663 * This function should never be called. We need to have it to pass
1664 * it as an argument to other functions.
1665 */
1666 /*ARGSUSED*/
1667 int
1668 hsfs_putapage(
1669 vnode_t *vp,
1670 page_t *pp,
1671 u_offset_t *offp,
1672 size_t *lenp,
1673 int flags,
1674 cred_t *cr)
1675 {
1676 /* should never happen - just destroy it */
1677 cmn_err(CE_NOTE, "hsfs_putapage: dirty HSFS page");
1678 pvn_write_done(pp, B_ERROR | B_WRITE | B_INVAL | B_FORCE | flags);
1679 return (0);
1680 }
1681
1682
1683 /*
1684 * The only flags we support are B_INVAL, B_FREE and B_DONTNEED.
1685 * B_INVAL is set by:
1686 *
1687 * 1) the MC_SYNC command of memcntl(2) to support the MS_INVALIDATE flag.
1688 * 2) the MC_ADVISE command of memcntl(2) with the MADV_DONTNEED advice
1689 * which translates to an MC_SYNC with the MS_INVALIDATE flag.
1690 *
1691 * The B_FREE (as well as the B_DONTNEED) flag is set when the
1692 * MADV_SEQUENTIAL advice has been used. VOP_PUTPAGE is invoked
1693 * from SEGVN to release pages behind a pagefault.
1694 */
1695 /*ARGSUSED*/
1696 static int
1697 hsfs_putpage(
1698 struct vnode *vp,
1699 offset_t off,
1700 size_t len,
1701 int flags,
1702 struct cred *cr,
1703 caller_context_t *ct)
1704 {
1705 int error = 0;
1706
1707 if (vp->v_count == 0) {
1708 panic("hsfs_putpage: bad v_count");
1709 /*NOTREACHED*/
1710 }
1711
1712 if (vp->v_flag & VNOMAP)
1713 return (ENOSYS);
1714
1715 ASSERT(off <= HS_MAXFILEOFF);
1716
1717 if (!vn_has_cached_data(vp)) /* no pages mapped */
1718 return (0);
1719
1720 if (len == 0) { /* from 'off' to EOF */
1721 error = pvn_vplist_dirty(vp, off, hsfs_putapage, flags, cr);
1722 } else {
1723 offset_t end_off = off + len;
1724 offset_t file_size = VTOH(vp)->hs_dirent.ext_size;
1725 offset_t io_off;
1726
1727 file_size = (file_size + PAGESIZE - 1) & PAGEMASK;
1728 if (end_off > file_size)
1729 end_off = file_size;
1730
1731 for (io_off = off; io_off < end_off; io_off += PAGESIZE) {
1732 page_t *pp;
1733
1734 /*
1735 * We insist on getting the page only if we are
1736 * about to invalidate, free or write it and
1737 * the B_ASYNC flag is not set.
1738 */
1739 if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
1740 pp = page_lookup(vp, io_off,
1741 (flags & (B_INVAL | B_FREE)) ?
1742 SE_EXCL : SE_SHARED);
1743 } else {
1744 pp = page_lookup_nowait(vp, io_off,
1745 (flags & B_FREE) ? SE_EXCL : SE_SHARED);
1746 }
1747
1748 if (pp == NULL)
1749 continue;
1750
1751 /*
1752 * Normally pvn_getdirty() should return 0, which
1753 * impies that it has done the job for us.
1754 * The shouldn't-happen scenario is when it returns 1.
1755 * This means that the page has been modified and
1756 * needs to be put back.
1757 * Since we can't write on a CD, we fake a failed
1758 * I/O and force pvn_write_done() to destroy the page.
1759 */
1760 if (pvn_getdirty(pp, flags) == 1) {
1761 cmn_err(CE_NOTE,
1762 "hsfs_putpage: dirty HSFS page");
1763 pvn_write_done(pp, flags |
1764 B_ERROR | B_WRITE | B_INVAL | B_FORCE);
1765 }
1766 }
1767 }
1768 return (error);
1769 }
1770
1771
1772 /*ARGSUSED*/
1773 static int
1774 hsfs_map(
1775 struct vnode *vp,
1776 offset_t off,
1777 struct as *as,
1778 caddr_t *addrp,
1779 size_t len,
1780 uchar_t prot,
1781 uchar_t maxprot,
1782 uint_t flags,
1783 struct cred *cred,
1784 caller_context_t *ct)
1785 {
1786 struct segvn_crargs vn_a;
1787 int error;
1788
1789 /* VFS_RECORD(vp->v_vfsp, VS_MAP, VS_CALL); */
1790
1791 if (vp->v_flag & VNOMAP)
1792 return (ENOSYS);
1793
1794 if ((prot & PROT_WRITE) && (flags & MAP_SHARED))
1795 return (ENOSYS);
1796
1797 if (off > HS_MAXFILEOFF || off < 0 ||
1798 (off + len) < 0 || (off + len) > HS_MAXFILEOFF)
1799 return (ENXIO);
1800
1801 if (vp->v_type != VREG) {
1802 return (ENODEV);
1803 }
1804
1805 /*
1806 * If file is being locked, disallow mapping.
1807 */
1808 if (vn_has_mandatory_locks(vp, VTOH(vp)->hs_dirent.mode))
1809 return (EAGAIN);
1810
1811 as_rangelock(as);
1812 error = choose_addr(as, addrp, len, off, ADDR_VACALIGN, flags);
1813 if (error != 0) {
1814 as_rangeunlock(as);
1815 return (error);
1816 }
1817
1818 vn_a.vp = vp;
1819 vn_a.offset = off;
1820 vn_a.type = flags & MAP_TYPE;
1821 vn_a.prot = prot;
1822 vn_a.maxprot = maxprot;
1823 vn_a.flags = flags & ~MAP_TYPE;
1824 vn_a.cred = cred;
1825 vn_a.amp = NULL;
1826 vn_a.szc = 0;
1827 vn_a.lgrp_mem_policy_flags = 0;
1828
1829 error = as_map(as, *addrp, len, segvn_create, &vn_a);
1830 as_rangeunlock(as);
1831 return (error);
1832 }
1833
1834 /* ARGSUSED */
1835 static int
1836 hsfs_addmap(
1837 struct vnode *vp,
1838 offset_t off,
1839 struct as *as,
1840 caddr_t addr,
1841 size_t len,
1842 uchar_t prot,
1843 uchar_t maxprot,
1844 uint_t flags,
1845 struct cred *cr,
1846 caller_context_t *ct)
1847 {
1848 struct hsnode *hp;
1849
1850 if (vp->v_flag & VNOMAP)
1851 return (ENOSYS);
1852
1853 hp = VTOH(vp);
1854 mutex_enter(&hp->hs_contents_lock);
1855 hp->hs_mapcnt += btopr(len);
1856 mutex_exit(&hp->hs_contents_lock);
1857 return (0);
1858 }
1859
1860 /*ARGSUSED*/
1861 static int
1862 hsfs_delmap(
1863 struct vnode *vp,
1864 offset_t off,
1865 struct as *as,
1866 caddr_t addr,
1867 size_t len,
1868 uint_t prot,
1869 uint_t maxprot,
1870 uint_t flags,
1871 struct cred *cr,
1872 caller_context_t *ct)
1873 {
1874 struct hsnode *hp;
1875
1876 if (vp->v_flag & VNOMAP)
1877 return (ENOSYS);
1878
1879 hp = VTOH(vp);
1880 mutex_enter(&hp->hs_contents_lock);
1881 hp->hs_mapcnt -= btopr(len); /* Count released mappings */
1882 ASSERT(hp->hs_mapcnt >= 0);
1883 mutex_exit(&hp->hs_contents_lock);
1884 return (0);
1885 }
1886
1887 /* ARGSUSED */
1888 static int
1889 hsfs_seek(
1890 struct vnode *vp,
1891 offset_t ooff,
1892 offset_t *noffp,
1893 caller_context_t *ct)
1894 {
1895 return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
1896 }
1897
1898 /* ARGSUSED */
1899 static int
1900 hsfs_frlock(
1901 struct vnode *vp,
1902 int cmd,
1903 struct flock64 *bfp,
1904 int flag,
1905 offset_t offset,
1906 struct flk_callback *flk_cbp,
1907 cred_t *cr,
1908 caller_context_t *ct)
1909 {
1910 struct hsnode *hp = VTOH(vp);
1911
1912 /*
1913 * If the file is being mapped, disallow fs_frlock.
1914 * We are not holding the hs_contents_lock while checking
1915 * hs_mapcnt because the current locking strategy drops all
1916 * locks before calling fs_frlock.
1917 * So, hs_mapcnt could change before we enter fs_frlock making
1918 * it meaningless to have held hs_contents_lock in the first place.
1919 */
1920 if (hp->hs_mapcnt > 0 && MANDLOCK(vp, hp->hs_dirent.mode))
1921 return (EAGAIN);
1922
1923 return (fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr, ct));
1924 }
1925
1926 static int
1927 hsched_deadline_compare(const void *x1, const void *x2)
1928 {
1929 const struct hio *h1 = x1;
1930 const struct hio *h2 = x2;
1931
1932 if (h1->io_timestamp < h2->io_timestamp)
1933 return (-1);
1934 if (h1->io_timestamp > h2->io_timestamp)
1935 return (1);
1936
1937 if (h1->io_lblkno < h2->io_lblkno)
1938 return (-1);
1939 if (h1->io_lblkno > h2->io_lblkno)
1940 return (1);
1941
1942 if (h1 < h2)
1943 return (-1);
1944 if (h1 > h2)
1945 return (1);
1946
1947 return (0);
1948 }
1949
1950 static int
1951 hsched_offset_compare(const void *x1, const void *x2)
1952 {
1953 const struct hio *h1 = x1;
1954 const struct hio *h2 = x2;
1955
1956 if (h1->io_lblkno < h2->io_lblkno)
1957 return (-1);
1958 if (h1->io_lblkno > h2->io_lblkno)
1959 return (1);
1960
1961 if (h1 < h2)
1962 return (-1);
1963 if (h1 > h2)
1964 return (1);
1965
1966 return (0);
1967 }
1968
1969 void
1970 hsched_init_caches(void)
1971 {
1972 hio_cache = kmem_cache_create("hsfs_hio_cache",
1973 sizeof (struct hio), 0, NULL,
1974 NULL, NULL, NULL, NULL, 0);
1975
1976 hio_info_cache = kmem_cache_create("hsfs_hio_info_cache",
1977 sizeof (struct hio_info), 0, NULL,
1978 NULL, NULL, NULL, NULL, 0);
1979 }
1980
1981 void
1982 hsched_fini_caches(void)
1983 {
1984 kmem_cache_destroy(hio_cache);
1985 kmem_cache_destroy(hio_info_cache);
1986 }
1987
1988 /*
1989 * Initialize I/O scheduling structures. This is called via hsfs_mount
1990 */
1991 void
1992 hsched_init(struct hsfs *fsp, int fsid, struct modlinkage *modlinkage)
1993 {
1994 struct hsfs_queue *hqueue = fsp->hqueue;
1995 struct vnode *vp = fsp->hsfs_devvp;
1996
1997 /* TaskQ name of the form: hsched_task_ + stringof(int) */
1998 char namebuf[23];
1999 int error, err;
2000 struct dk_cinfo info;
2001 ldi_handle_t lh;
2002 ldi_ident_t li;
2003
2004 /*
2005 * Default maxtransfer = 16k chunk
2006 */
2007 hqueue->dev_maxtransfer = 16384;
2008
2009 /*
2010 * Try to fetch the maximum device transfer size. This is used to
2011 * ensure that a coalesced block does not exceed the maxtransfer.
2012 */
2013 err = ldi_ident_from_mod(modlinkage, &li);
2014 if (err) {
2015 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2016 cmn_err(CE_NOTE, "hsched_init: ldi_ident_from_mod err=%d\n",
2017 err);
2018 goto set_ra;
2019 }
2020
2021 err = ldi_open_by_dev(&(vp->v_rdev), OTYP_CHR, FREAD, CRED(), &lh, li);
2022 ldi_ident_release(li);
2023 if (err) {
2024 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2025 cmn_err(CE_NOTE, "hsched_init: ldi_open err=%d\n", err);
2026 goto set_ra;
2027 }
2028
2029 error = ldi_ioctl(lh, DKIOCINFO, (intptr_t)&info, FKIOCTL,
2030 CRED(), &err);
2031 err = ldi_close(lh, FREAD, CRED());
2032 if (err) {
2033 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2034 cmn_err(CE_NOTE, "hsched_init: ldi_close err=%d\n", err);
2035 }
2036
2037 if (error == 0) {
2038 hqueue->dev_maxtransfer = ldbtob(info.dki_maxtransfer);
2039 }
2040
2041 set_ra:
2042 /*
2043 * Max size of data to read ahead for sequential access pattern.
2044 * Conservative to avoid letting the underlying CD drive to spin
2045 * down, in case the application is reading slowly.
2046 * We read ahead upto a max of 4 pages.
2047 */
2048 hqueue->max_ra_bytes = PAGESIZE * 8;
2049
2050 mutex_init(&(hqueue->hsfs_queue_lock), NULL, MUTEX_DEFAULT, NULL);
2051 mutex_init(&(hqueue->strategy_lock), NULL, MUTEX_DEFAULT, NULL);
2052 avl_create(&(hqueue->read_tree), hsched_offset_compare,
2053 sizeof (struct hio), offsetof(struct hio, io_offset_node));
2054 avl_create(&(hqueue->deadline_tree), hsched_deadline_compare,
2055 sizeof (struct hio), offsetof(struct hio, io_deadline_node));
2056
2057 (void) snprintf(namebuf, sizeof (namebuf), "hsched_task_%d", fsid);
2058 hqueue->ra_task = taskq_create(namebuf, hsfs_taskq_nthreads,
2059 minclsyspri + 2, 1, 104857600 / PAGESIZE, TASKQ_DYNAMIC);
2060
2061 hqueue->next = NULL;
2062 hqueue->nbuf = kmem_zalloc(sizeof (struct buf), KM_SLEEP);
2063 }
2064
2065 void
2066 hsched_fini(struct hsfs_queue *hqueue)
2067 {
2068 if (hqueue != NULL) {
2069 /*
2070 * Remove the sentinel if there was one.
2071 */
2072 if (hqueue->next != NULL) {
2073 avl_remove(&hqueue->read_tree, hqueue->next);
2074 kmem_cache_free(hio_cache, hqueue->next);
2075 }
2076 avl_destroy(&(hqueue->read_tree));
2077 avl_destroy(&(hqueue->deadline_tree));
2078 mutex_destroy(&(hqueue->hsfs_queue_lock));
2079 mutex_destroy(&(hqueue->strategy_lock));
2080
2081 /*
2082 * If there are any existing readahead threads running
2083 * taskq_destroy will wait for them to finish.
2084 */
2085 taskq_destroy(hqueue->ra_task);
2086 kmem_free(hqueue->nbuf, sizeof (struct buf));
2087 }
2088 }
2089
2090 /*
2091 * Determine if two I/O requests are adjacent to each other so
2092 * that they can coalesced.
2093 */
2094 #define IS_ADJACENT(io, nio) \
2095 (((io)->io_lblkno + (io)->nblocks == (nio)->io_lblkno) && \
2096 (io)->bp->b_edev == (nio)->bp->b_edev)
2097
2098 /*
2099 * This performs the actual I/O scheduling logic. We use the Circular
2100 * Look algorithm here. Sort the I/O requests in ascending order of
2101 * logical block number and process them starting with the lowest
2102 * numbered block and progressing towards higher block numbers in the
2103 * queue. Once there are no more higher numbered blocks, start again
2104 * with the lowest one. This is good for CD/DVD as you keep moving
2105 * the head in one direction along the outward spiral track and avoid
2106 * too many seeks as much as possible. The re-ordering also allows
2107 * us to coalesce adjacent requests into one larger request.
2108 * This is thus essentially a 1-way Elevator with front merging.
2109 *
2110 * In addition each read request here has a deadline and will be
2111 * processed out of turn if the deadline (500ms) expires.
2112 *
2113 * This function is necessarily serialized via hqueue->strategy_lock.
2114 * This function sits just below hsfs_getapage and processes all read
2115 * requests orginating from that function.
2116 */
2117 int
2118 hsched_invoke_strategy(struct hsfs *fsp)
2119 {
2120 struct hsfs_queue *hqueue;
2121 struct buf *nbuf;
2122 struct hio *fio, *nio, *tio, *prev, *last;
2123 size_t bsize, soffset, offset, data;
2124 int bioret, bufcount;
2125 struct vnode *fvp;
2126 ksema_t *io_done;
2127 caddr_t iodata;
2128
2129 hqueue = fsp->hqueue;
2130 mutex_enter(&hqueue->strategy_lock);
2131 mutex_enter(&hqueue->hsfs_queue_lock);
2132
2133 /*
2134 * Check for Deadline expiration first
2135 */
2136 fio = avl_first(&hqueue->deadline_tree);
2137
2138 /*
2139 * Paranoid check for empty I/O queue. Both deadline
2140 * and read trees contain same data sorted in different
2141 * ways. So empty deadline tree = empty read tree.
2142 */
2143 if (fio == NULL) {
2144 /*
2145 * Remove the sentinel if there was one.
2146 */
2147 if (hqueue->next != NULL) {
2148 avl_remove(&hqueue->read_tree, hqueue->next);
2149 kmem_cache_free(hio_cache, hqueue->next);
2150 hqueue->next = NULL;
2151 }
2152 mutex_exit(&hqueue->hsfs_queue_lock);
2153 mutex_exit(&hqueue->strategy_lock);
2154 return (1);
2155 }
2156
2157 if (drv_hztousec(ddi_get_lbolt()) - fio->io_timestamp
2158 < HSFS_READ_DEADLINE) {
2159 /*
2160 * Apply standard scheduling logic. This uses the
2161 * C-LOOK approach. Process I/O requests in ascending
2162 * order of logical block address till no subsequent
2163 * higher numbered block request remains. Then start
2164 * again from the lowest numbered block in the queue.
2165 *
2166 * We do this cheaply here by means of a sentinel.
2167 * The last processed I/O structure from the previous
2168 * invocation of this func, is left dangling in the
2169 * read_tree so that we can easily scan to the next
2170 * higher numbered request and remove the sentinel.
2171 */
2172 fio = NULL;
2173 if (hqueue->next != NULL) {
2174 fio = AVL_NEXT(&hqueue->read_tree, hqueue->next);
2175 avl_remove(&hqueue->read_tree, hqueue->next);
2176 kmem_cache_free(hio_cache, hqueue->next);
2177 hqueue->next = NULL;
2178 }
2179 if (fio == NULL) {
2180 fio = avl_first(&hqueue->read_tree);
2181 }
2182 } else if (hqueue->next != NULL) {
2183 DTRACE_PROBE1(hsfs_deadline_expiry, struct hio *, fio);
2184
2185 avl_remove(&hqueue->read_tree, hqueue->next);
2186 kmem_cache_free(hio_cache, hqueue->next);
2187 hqueue->next = NULL;
2188 }
2189
2190 /*
2191 * In addition we try to coalesce contiguous
2192 * requests into one bigger request.
2193 */
2194 bufcount = 1;
2195 bsize = ldbtob(fio->nblocks);
2196 fvp = fio->bp->b_file;
2197 nio = AVL_NEXT(&hqueue->read_tree, fio);
2198 tio = fio;
2199 while (nio != NULL && IS_ADJACENT(tio, nio) &&
2200 bsize < hqueue->dev_maxtransfer) {
2201 avl_remove(&hqueue->deadline_tree, tio);
2202 avl_remove(&hqueue->read_tree, tio);
2203 tio->contig_chain = nio;
2204 bsize += ldbtob(nio->nblocks);
2205 prev = tio;
2206 tio = nio;
2207
2208 /*
2209 * This check is required to detect the case where
2210 * we are merging adjacent buffers belonging to
2211 * different files. fvp is used to set the b_file
2212 * parameter in the coalesced buf. b_file is used
2213 * by DTrace so we do not want DTrace to accrue
2214 * requests to two different files to any one file.
2215 */
2216 if (fvp && tio->bp->b_file != fvp) {
2217 fvp = NULL;
2218 }
2219
2220 nio = AVL_NEXT(&hqueue->read_tree, nio);
2221 bufcount++;
2222 }
2223
2224 /*
2225 * tio is not removed from the read_tree as it serves as a sentinel
2226 * to cheaply allow us to scan to the next higher numbered I/O
2227 * request.
2228 */
2229 hqueue->next = tio;
2230 avl_remove(&hqueue->deadline_tree, tio);
2231 mutex_exit(&hqueue->hsfs_queue_lock);
2232 DTRACE_PROBE3(hsfs_io_dequeued, struct hio *, fio, int, bufcount,
2233 size_t, bsize);
2234
2235 /*
2236 * The benefit of coalescing occurs if the the savings in I/O outweighs
2237 * the cost of doing the additional work below.
2238 * It was observed that coalescing 2 buffers results in diminishing
2239 * returns, so we do coalescing if we have >2 adjacent bufs.
2240 */
2241 if (bufcount > hsched_coalesce_min) {
2242 /*
2243 * We have coalesced blocks. First allocate mem and buf for
2244 * the entire coalesced chunk.
2245 * Since we are guaranteed single-threaded here we pre-allocate
2246 * one buf at mount time and that is re-used every time. This
2247 * is a synthesized buf structure that uses kmem_alloced chunk.
2248 * Not quite a normal buf attached to pages.
2249 */
2250 fsp->coalesced_bytes += bsize;
2251 nbuf = hqueue->nbuf;
2252 bioinit(nbuf);
2253 nbuf->b_edev = fio->bp->b_edev;
2254 nbuf->b_dev = fio->bp->b_dev;
2255 nbuf->b_flags = fio->bp->b_flags;
2256 nbuf->b_iodone = fio->bp->b_iodone;
2257 iodata = kmem_alloc(bsize, KM_SLEEP);
2258 nbuf->b_un.b_addr = iodata;
2259 nbuf->b_lblkno = fio->bp->b_lblkno;
2260 nbuf->b_vp = fvp;
2261 nbuf->b_file = fvp;
2262 nbuf->b_bcount = bsize;
2263 nbuf->b_bufsize = bsize;
2264
2265 DTRACE_PROBE3(hsfs_coalesced_io_start, struct hio *, fio, int,
2266 bufcount, size_t, bsize);
2267
2268 /*
2269 * Perform I/O for the coalesced block.
2270 */
2271 (void) bdev_strategy(nbuf);
2272
2273 /*
2274 * Duplicate the last IO node to leave the sentinel alone.
2275 * The sentinel is freed in the next invocation of this
2276 * function.
2277 */
2278 prev->contig_chain = kmem_cache_alloc(hio_cache, KM_SLEEP);
2279 prev->contig_chain->bp = tio->bp;
2280 prev->contig_chain->sema = tio->sema;
2281 tio = prev->contig_chain;
2282 tio->contig_chain = NULL;
2283 soffset = ldbtob(fio->bp->b_lblkno);
2284 nio = fio;
2285
2286 bioret = biowait(nbuf);
2287 data = bsize - nbuf->b_resid;
2288 biofini(nbuf);
2289 mutex_exit(&hqueue->strategy_lock);
2290
2291 /*
2292 * We use the b_resid parameter to detect how much
2293 * data was succesfully transferred. We will signal
2294 * a success to all the fully retrieved actual bufs
2295 * before coalescing, rest is signaled as error,
2296 * if any.
2297 */
2298 tio = nio;
2299 DTRACE_PROBE3(hsfs_coalesced_io_done, struct hio *, nio,
2300 int, bioret, size_t, data);
2301
2302 /*
2303 * Copy data and signal success to all the bufs
2304 * which can be fully satisfied from b_resid.
2305 */
2306 while (nio != NULL && data >= nio->bp->b_bcount) {
2307 offset = ldbtob(nio->bp->b_lblkno) - soffset;
2308 bcopy(iodata + offset, nio->bp->b_un.b_addr,
2309 nio->bp->b_bcount);
2310 data -= nio->bp->b_bcount;
2311 bioerror(nio->bp, 0);
2312 biodone(nio->bp);
2313 sema_v(nio->sema);
2314 tio = nio;
2315 nio = nio->contig_chain;
2316 kmem_cache_free(hio_cache, tio);
2317 }
2318
2319 /*
2320 * Signal error to all the leftover bufs (if any)
2321 * after b_resid data is exhausted.
2322 */
2323 while (nio != NULL) {
2324 nio->bp->b_resid = nio->bp->b_bcount - data;
2325 bzero(nio->bp->b_un.b_addr + data, nio->bp->b_resid);
2326 bioerror(nio->bp, bioret);
2327 biodone(nio->bp);
2328 sema_v(nio->sema);
2329 tio = nio;
2330 nio = nio->contig_chain;
2331 kmem_cache_free(hio_cache, tio);
2332 data = 0;
2333 }
2334 kmem_free(iodata, bsize);
2335 } else {
2336
2337 nbuf = tio->bp;
2338 io_done = tio->sema;
2339 nio = fio;
2340 last = tio;
2341
2342 while (nio != NULL) {
2343 (void) bdev_strategy(nio->bp);
2344 nio = nio->contig_chain;
2345 }
2346 nio = fio;
2347 mutex_exit(&hqueue->strategy_lock);
2348
2349 while (nio != NULL) {
2350 if (nio == last) {
2351 (void) biowait(nbuf);
2352 sema_v(io_done);
2353 break;
2354 /* sentinel last not freed. See above. */
2355 } else {
2356 (void) biowait(nio->bp);
2357 sema_v(nio->sema);
2358 }
2359 tio = nio;
2360 nio = nio->contig_chain;
2361 kmem_cache_free(hio_cache, tio);
2362 }
2363 }
2364 return (0);
2365 }
2366
2367 /*
2368 * Insert an I/O request in the I/O scheduler's pipeline
2369 * Using AVL tree makes it easy to reorder the I/O request
2370 * based on logical block number.
2371 */
2372 static void
2373 hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra)
2374 {
2375 struct hsfs_queue *hqueue = fsp->hqueue;
2376
2377 mutex_enter(&hqueue->hsfs_queue_lock);
2378
2379 fsp->physical_read_bytes += hsio->bp->b_bcount;
2380 if (ra)
2381 fsp->readahead_bytes += hsio->bp->b_bcount;
2382
2383 avl_add(&hqueue->deadline_tree, hsio);
2384 avl_add(&hqueue->read_tree, hsio);
2385
2386 DTRACE_PROBE3(hsfs_io_enqueued, struct hio *, hsio,
2387 struct hsfs_queue *, hqueue, int, ra);
2388
2389 mutex_exit(&hqueue->hsfs_queue_lock);
2390 }
2391
2392 /* ARGSUSED */
2393 static int
2394 hsfs_pathconf(struct vnode *vp,
2395 int cmd,
2396 ulong_t *valp,
2397 struct cred *cr,
2398 caller_context_t *ct)
2399 {
2400 struct hsfs *fsp;
2401
2402 int error = 0;
2403
2404 switch (cmd) {
2405
2406 case _PC_NAME_MAX:
2407 fsp = VFS_TO_HSFS(vp->v_vfsp);
2408 *valp = fsp->hsfs_namemax;
2409 break;
2410
2411 case _PC_FILESIZEBITS:
2412 *valp = 33; /* Without multi extent support: 4 GB - 2k */
2413 break;
2414
2415 case _PC_TIMESTAMP_RESOLUTION:
2416 /*
2417 * HSFS keeps, at best, 1/100 second timestamp resolution.
2418 */
2419 *valp = 10000000L;
2420 break;
2421
2422 default:
2423 error = fs_pathconf(vp, cmd, valp, cr, ct);
2424 break;
2425 }
2426
2427 return (error);
2428 }
2429
2430
2431
2432 const fs_operation_def_t hsfs_vnodeops_template[] = {
2433 VOPNAME_OPEN, { .vop_open = hsfs_open },
2434 VOPNAME_CLOSE, { .vop_close = hsfs_close },
2435 VOPNAME_READ, { .vop_read = hsfs_read },
2436 VOPNAME_GETATTR, { .vop_getattr = hsfs_getattr },
2437 VOPNAME_ACCESS, { .vop_access = hsfs_access },
2438 VOPNAME_LOOKUP, { .vop_lookup = hsfs_lookup },
2439 VOPNAME_READDIR, { .vop_readdir = hsfs_readdir },
2440 VOPNAME_READLINK, { .vop_readlink = hsfs_readlink },
2441 VOPNAME_FSYNC, { .vop_fsync = hsfs_fsync },
2442 VOPNAME_INACTIVE, { .vop_inactive = hsfs_inactive },
2443 VOPNAME_FID, { .vop_fid = hsfs_fid },
2444 VOPNAME_SEEK, { .vop_seek = hsfs_seek },
2445 VOPNAME_FRLOCK, { .vop_frlock = hsfs_frlock },
2446 VOPNAME_GETPAGE, { .vop_getpage = hsfs_getpage },
2447 VOPNAME_PUTPAGE, { .vop_putpage = hsfs_putpage },
2448 VOPNAME_MAP, { .vop_map = hsfs_map },
2449 VOPNAME_ADDMAP, { .vop_addmap = hsfs_addmap },
2450 VOPNAME_DELMAP, { .vop_delmap = hsfs_delmap },
2451 VOPNAME_PATHCONF, { .vop_pathconf = hsfs_pathconf },
2452 NULL, NULL
2453 };
2454
2455 struct vnodeops *hsfs_vnodeops;