Print this page
4347 ZPL can use dmu_tx_assign(TXG_WAIT)
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/zfs_dir.c
+++ new/usr/src/uts/common/fs/zfs/zfs_dir.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright (c) 2013 by Delphix. All rights reserved.
24 24 */
25 25
26 26 #include <sys/types.h>
27 27 #include <sys/param.h>
28 28 #include <sys/time.h>
29 29 #include <sys/systm.h>
30 30 #include <sys/sysmacros.h>
31 31 #include <sys/resource.h>
32 32 #include <sys/vfs.h>
33 33 #include <sys/vnode.h>
34 34 #include <sys/file.h>
35 35 #include <sys/mode.h>
36 36 #include <sys/kmem.h>
37 37 #include <sys/uio.h>
38 38 #include <sys/pathname.h>
39 39 #include <sys/cmn_err.h>
40 40 #include <sys/errno.h>
41 41 #include <sys/stat.h>
42 42 #include <sys/unistd.h>
43 43 #include <sys/sunddi.h>
44 44 #include <sys/random.h>
45 45 #include <sys/policy.h>
46 46 #include <sys/zfs_dir.h>
47 47 #include <sys/zfs_acl.h>
48 48 #include <sys/fs/zfs.h>
49 49 #include "fs/fs_subr.h"
50 50 #include <sys/zap.h>
51 51 #include <sys/dmu.h>
52 52 #include <sys/atomic.h>
53 53 #include <sys/zfs_ctldir.h>
54 54 #include <sys/zfs_fuid.h>
55 55 #include <sys/sa.h>
56 56 #include <sys/zfs_sa.h>
57 57 #include <sys/dnlc.h>
58 58 #include <sys/extdirent.h>
59 59
60 60 /*
61 61 * zfs_match_find() is used by zfs_dirent_lock() to peform zap lookups
62 62 * of names after deciding which is the appropriate lookup interface.
63 63 */
64 64 static int
65 65 zfs_match_find(zfsvfs_t *zfsvfs, znode_t *dzp, char *name, boolean_t exact,
66 66 boolean_t update, int *deflags, pathname_t *rpnp, uint64_t *zoid)
67 67 {
68 68 int error;
69 69
70 70 if (zfsvfs->z_norm) {
71 71 matchtype_t mt = MT_FIRST;
72 72 boolean_t conflict = B_FALSE;
73 73 size_t bufsz = 0;
74 74 char *buf = NULL;
75 75
76 76 if (rpnp) {
77 77 buf = rpnp->pn_buf;
78 78 bufsz = rpnp->pn_bufsize;
79 79 }
80 80 if (exact)
81 81 mt = MT_EXACT;
82 82 /*
83 83 * In the non-mixed case we only expect there would ever
84 84 * be one match, but we need to use the normalizing lookup.
85 85 */
86 86 error = zap_lookup_norm(zfsvfs->z_os, dzp->z_id, name, 8, 1,
87 87 zoid, mt, buf, bufsz, &conflict);
88 88 if (!error && deflags)
89 89 *deflags = conflict ? ED_CASE_CONFLICT : 0;
90 90 } else {
91 91 error = zap_lookup(zfsvfs->z_os, dzp->z_id, name, 8, 1, zoid);
92 92 }
93 93 *zoid = ZFS_DIRENT_OBJ(*zoid);
94 94
95 95 if (error == ENOENT && update)
96 96 dnlc_update(ZTOV(dzp), name, DNLC_NO_VNODE);
97 97
98 98 return (error);
99 99 }
100 100
101 101 /*
102 102 * Lock a directory entry. A dirlock on <dzp, name> protects that name
103 103 * in dzp's directory zap object. As long as you hold a dirlock, you can
104 104 * assume two things: (1) dzp cannot be reaped, and (2) no other thread
105 105 * can change the zap entry for (i.e. link or unlink) this name.
106 106 *
107 107 * Input arguments:
108 108 * dzp - znode for directory
109 109 * name - name of entry to lock
110 110 * flag - ZNEW: if the entry already exists, fail with EEXIST.
111 111 * ZEXISTS: if the entry does not exist, fail with ENOENT.
112 112 * ZSHARED: allow concurrent access with other ZSHARED callers.
113 113 * ZXATTR: we want dzp's xattr directory
114 114 * ZCILOOK: On a mixed sensitivity file system,
115 115 * this lookup should be case-insensitive.
116 116 * ZCIEXACT: On a purely case-insensitive file system,
117 117 * this lookup should be case-sensitive.
118 118 * ZRENAMING: we are locking for renaming, force narrow locks
119 119 * ZHAVELOCK: Don't grab the z_name_lock for this call. The
120 120 * current thread already holds it.
121 121 *
122 122 * Output arguments:
123 123 * zpp - pointer to the znode for the entry (NULL if there isn't one)
124 124 * dlpp - pointer to the dirlock for this entry (NULL on error)
125 125 * direntflags - (case-insensitive lookup only)
126 126 * flags if multiple case-sensitive matches exist in directory
127 127 * realpnp - (case-insensitive lookup only)
128 128 * actual name matched within the directory
129 129 *
130 130 * Return value: 0 on success or errno on failure.
131 131 *
132 132 * NOTE: Always checks for, and rejects, '.' and '..'.
133 133 * NOTE: For case-insensitive file systems we take wide locks (see below),
134 134 * but return znode pointers to a single match.
135 135 */
136 136 int
137 137 zfs_dirent_lock(zfs_dirlock_t **dlpp, znode_t *dzp, char *name, znode_t **zpp,
138 138 int flag, int *direntflags, pathname_t *realpnp)
139 139 {
140 140 zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
141 141 zfs_dirlock_t *dl;
142 142 boolean_t update;
143 143 boolean_t exact;
144 144 uint64_t zoid;
145 145 vnode_t *vp = NULL;
146 146 int error = 0;
147 147 int cmpflags;
148 148
149 149 *zpp = NULL;
150 150 *dlpp = NULL;
151 151
152 152 /*
153 153 * Verify that we are not trying to lock '.', '..', or '.zfs'
154 154 */
155 155 if (name[0] == '.' &&
156 156 (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')) ||
157 157 zfs_has_ctldir(dzp) && strcmp(name, ZFS_CTLDIR_NAME) == 0)
158 158 return (SET_ERROR(EEXIST));
159 159
160 160 /*
161 161 * Case sensitivity and normalization preferences are set when
162 162 * the file system is created. These are stored in the
163 163 * zfsvfs->z_case and zfsvfs->z_norm fields. These choices
164 164 * affect what vnodes can be cached in the DNLC, how we
165 165 * perform zap lookups, and the "width" of our dirlocks.
166 166 *
167 167 * A normal dirlock locks a single name. Note that with
168 168 * normalization a name can be composed multiple ways, but
169 169 * when normalized, these names all compare equal. A wide
170 170 * dirlock locks multiple names. We need these when the file
171 171 * system is supporting mixed-mode access. It is sometimes
172 172 * necessary to lock all case permutations of file name at
173 173 * once so that simultaneous case-insensitive/case-sensitive
174 174 * behaves as rationally as possible.
175 175 */
176 176
177 177 /*
178 178 * Decide if exact matches should be requested when performing
179 179 * a zap lookup on file systems supporting case-insensitive
180 180 * access.
181 181 */
182 182 exact =
183 183 ((zfsvfs->z_case == ZFS_CASE_INSENSITIVE) && (flag & ZCIEXACT)) ||
184 184 ((zfsvfs->z_case == ZFS_CASE_MIXED) && !(flag & ZCILOOK));
185 185
186 186 /*
187 187 * Only look in or update the DNLC if we are looking for the
188 188 * name on a file system that does not require normalization
189 189 * or case folding. We can also look there if we happen to be
190 190 * on a non-normalizing, mixed sensitivity file system IF we
191 191 * are looking for the exact name.
192 192 *
193 193 * Maybe can add TO-UPPERed version of name to dnlc in ci-only
194 194 * case for performance improvement?
195 195 */
196 196 update = !zfsvfs->z_norm ||
197 197 ((zfsvfs->z_case == ZFS_CASE_MIXED) &&
198 198 !(zfsvfs->z_norm & ~U8_TEXTPREP_TOUPPER) && !(flag & ZCILOOK));
199 199
200 200 /*
201 201 * ZRENAMING indicates we are in a situation where we should
202 202 * take narrow locks regardless of the file system's
203 203 * preferences for normalizing and case folding. This will
204 204 * prevent us deadlocking trying to grab the same wide lock
205 205 * twice if the two names happen to be case-insensitive
206 206 * matches.
207 207 */
208 208 if (flag & ZRENAMING)
209 209 cmpflags = 0;
210 210 else
211 211 cmpflags = zfsvfs->z_norm;
212 212
213 213 /*
214 214 * Wait until there are no locks on this name.
215 215 *
216 216 * Don't grab the the lock if it is already held. However, cannot
217 217 * have both ZSHARED and ZHAVELOCK together.
218 218 */
219 219 ASSERT(!(flag & ZSHARED) || !(flag & ZHAVELOCK));
220 220 if (!(flag & ZHAVELOCK))
221 221 rw_enter(&dzp->z_name_lock, RW_READER);
222 222
223 223 mutex_enter(&dzp->z_lock);
224 224 for (;;) {
225 225 if (dzp->z_unlinked) {
226 226 mutex_exit(&dzp->z_lock);
227 227 if (!(flag & ZHAVELOCK))
228 228 rw_exit(&dzp->z_name_lock);
229 229 return (SET_ERROR(ENOENT));
230 230 }
231 231 for (dl = dzp->z_dirlocks; dl != NULL; dl = dl->dl_next) {
232 232 if ((u8_strcmp(name, dl->dl_name, 0, cmpflags,
233 233 U8_UNICODE_LATEST, &error) == 0) || error != 0)
234 234 break;
235 235 }
236 236 if (error != 0) {
237 237 mutex_exit(&dzp->z_lock);
238 238 if (!(flag & ZHAVELOCK))
239 239 rw_exit(&dzp->z_name_lock);
240 240 return (SET_ERROR(ENOENT));
241 241 }
242 242 if (dl == NULL) {
243 243 /*
244 244 * Allocate a new dirlock and add it to the list.
245 245 */
246 246 dl = kmem_alloc(sizeof (zfs_dirlock_t), KM_SLEEP);
247 247 cv_init(&dl->dl_cv, NULL, CV_DEFAULT, NULL);
248 248 dl->dl_name = name;
249 249 dl->dl_sharecnt = 0;
250 250 dl->dl_namelock = 0;
251 251 dl->dl_namesize = 0;
252 252 dl->dl_dzp = dzp;
253 253 dl->dl_next = dzp->z_dirlocks;
254 254 dzp->z_dirlocks = dl;
255 255 break;
256 256 }
257 257 if ((flag & ZSHARED) && dl->dl_sharecnt != 0)
258 258 break;
259 259 cv_wait(&dl->dl_cv, &dzp->z_lock);
260 260 }
261 261
262 262 /*
263 263 * If the z_name_lock was NOT held for this dirlock record it.
264 264 */
265 265 if (flag & ZHAVELOCK)
266 266 dl->dl_namelock = 1;
267 267
268 268 if ((flag & ZSHARED) && ++dl->dl_sharecnt > 1 && dl->dl_namesize == 0) {
269 269 /*
270 270 * We're the second shared reference to dl. Make a copy of
271 271 * dl_name in case the first thread goes away before we do.
272 272 * Note that we initialize the new name before storing its
273 273 * pointer into dl_name, because the first thread may load
274 274 * dl->dl_name at any time. He'll either see the old value,
275 275 * which is his, or the new shared copy; either is OK.
276 276 */
277 277 dl->dl_namesize = strlen(dl->dl_name) + 1;
278 278 name = kmem_alloc(dl->dl_namesize, KM_SLEEP);
279 279 bcopy(dl->dl_name, name, dl->dl_namesize);
280 280 dl->dl_name = name;
281 281 }
282 282
283 283 mutex_exit(&dzp->z_lock);
284 284
285 285 /*
286 286 * We have a dirlock on the name. (Note that it is the dirlock,
287 287 * not the dzp's z_lock, that protects the name in the zap object.)
288 288 * See if there's an object by this name; if so, put a hold on it.
289 289 */
290 290 if (flag & ZXATTR) {
291 291 error = sa_lookup(dzp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &zoid,
292 292 sizeof (zoid));
293 293 if (error == 0)
294 294 error = (zoid == 0 ? ENOENT : 0);
295 295 } else {
296 296 if (update)
297 297 vp = dnlc_lookup(ZTOV(dzp), name);
298 298 if (vp == DNLC_NO_VNODE) {
299 299 VN_RELE(vp);
300 300 error = SET_ERROR(ENOENT);
301 301 } else if (vp) {
302 302 if (flag & ZNEW) {
303 303 zfs_dirent_unlock(dl);
304 304 VN_RELE(vp);
305 305 return (SET_ERROR(EEXIST));
306 306 }
307 307 *dlpp = dl;
308 308 *zpp = VTOZ(vp);
309 309 return (0);
310 310 } else {
311 311 error = zfs_match_find(zfsvfs, dzp, name, exact,
312 312 update, direntflags, realpnp, &zoid);
313 313 }
314 314 }
315 315 if (error) {
316 316 if (error != ENOENT || (flag & ZEXISTS)) {
317 317 zfs_dirent_unlock(dl);
318 318 return (error);
319 319 }
320 320 } else {
321 321 if (flag & ZNEW) {
322 322 zfs_dirent_unlock(dl);
323 323 return (SET_ERROR(EEXIST));
324 324 }
325 325 error = zfs_zget(zfsvfs, zoid, zpp);
326 326 if (error) {
327 327 zfs_dirent_unlock(dl);
328 328 return (error);
329 329 }
330 330 if (!(flag & ZXATTR) && update)
331 331 dnlc_update(ZTOV(dzp), name, ZTOV(*zpp));
332 332 }
333 333
334 334 *dlpp = dl;
335 335
336 336 return (0);
337 337 }
338 338
339 339 /*
340 340 * Unlock this directory entry and wake anyone who was waiting for it.
341 341 */
342 342 void
343 343 zfs_dirent_unlock(zfs_dirlock_t *dl)
344 344 {
345 345 znode_t *dzp = dl->dl_dzp;
346 346 zfs_dirlock_t **prev_dl, *cur_dl;
347 347
348 348 mutex_enter(&dzp->z_lock);
349 349
350 350 if (!dl->dl_namelock)
351 351 rw_exit(&dzp->z_name_lock);
352 352
353 353 if (dl->dl_sharecnt > 1) {
354 354 dl->dl_sharecnt--;
355 355 mutex_exit(&dzp->z_lock);
356 356 return;
357 357 }
358 358 prev_dl = &dzp->z_dirlocks;
359 359 while ((cur_dl = *prev_dl) != dl)
360 360 prev_dl = &cur_dl->dl_next;
361 361 *prev_dl = dl->dl_next;
362 362 cv_broadcast(&dl->dl_cv);
363 363 mutex_exit(&dzp->z_lock);
364 364
365 365 if (dl->dl_namesize != 0)
366 366 kmem_free(dl->dl_name, dl->dl_namesize);
367 367 cv_destroy(&dl->dl_cv);
368 368 kmem_free(dl, sizeof (*dl));
369 369 }
370 370
371 371 /*
372 372 * Look up an entry in a directory.
373 373 *
374 374 * NOTE: '.' and '..' are handled as special cases because
375 375 * no directory entries are actually stored for them. If this is
376 376 * the root of a filesystem, then '.zfs' is also treated as a
377 377 * special pseudo-directory.
378 378 */
379 379 int
380 380 zfs_dirlook(znode_t *dzp, char *name, vnode_t **vpp, int flags,
381 381 int *deflg, pathname_t *rpnp)
382 382 {
383 383 zfs_dirlock_t *dl;
384 384 znode_t *zp;
385 385 int error = 0;
386 386 uint64_t parent;
387 387
388 388 if (name[0] == 0 || (name[0] == '.' && name[1] == 0)) {
389 389 *vpp = ZTOV(dzp);
390 390 VN_HOLD(*vpp);
391 391 } else if (name[0] == '.' && name[1] == '.' && name[2] == 0) {
392 392 zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
393 393
394 394 /*
395 395 * If we are a snapshot mounted under .zfs, return
396 396 * the vp for the snapshot directory.
397 397 */
398 398 if ((error = sa_lookup(dzp->z_sa_hdl,
399 399 SA_ZPL_PARENT(zfsvfs), &parent, sizeof (parent))) != 0)
400 400 return (error);
401 401 if (parent == dzp->z_id && zfsvfs->z_parent != zfsvfs) {
402 402 error = zfsctl_root_lookup(zfsvfs->z_parent->z_ctldir,
403 403 "snapshot", vpp, NULL, 0, NULL, kcred,
404 404 NULL, NULL, NULL);
405 405 return (error);
406 406 }
407 407 rw_enter(&dzp->z_parent_lock, RW_READER);
408 408 error = zfs_zget(zfsvfs, parent, &zp);
409 409 if (error == 0)
410 410 *vpp = ZTOV(zp);
411 411 rw_exit(&dzp->z_parent_lock);
412 412 } else if (zfs_has_ctldir(dzp) && strcmp(name, ZFS_CTLDIR_NAME) == 0) {
413 413 *vpp = zfsctl_root(dzp);
414 414 } else {
415 415 int zf;
416 416
417 417 zf = ZEXISTS | ZSHARED;
418 418 if (flags & FIGNORECASE)
419 419 zf |= ZCILOOK;
420 420
421 421 error = zfs_dirent_lock(&dl, dzp, name, &zp, zf, deflg, rpnp);
422 422 if (error == 0) {
423 423 *vpp = ZTOV(zp);
424 424 zfs_dirent_unlock(dl);
425 425 dzp->z_zn_prefetch = B_TRUE; /* enable prefetching */
426 426 }
427 427 rpnp = NULL;
428 428 }
429 429
430 430 if ((flags & FIGNORECASE) && rpnp && !error)
431 431 (void) strlcpy(rpnp->pn_buf, name, rpnp->pn_bufsize);
432 432
433 433 return (error);
434 434 }
435 435
436 436 /*
437 437 * unlinked Set (formerly known as the "delete queue") Error Handling
438 438 *
439 439 * When dealing with the unlinked set, we dmu_tx_hold_zap(), but we
440 440 * don't specify the name of the entry that we will be manipulating. We
441 441 * also fib and say that we won't be adding any new entries to the
442 442 * unlinked set, even though we might (this is to lower the minimum file
443 443 * size that can be deleted in a full filesystem). So on the small
444 444 * chance that the nlink list is using a fat zap (ie. has more than
445 445 * 2000 entries), we *may* not pre-read a block that's needed.
446 446 * Therefore it is remotely possible for some of the assertions
447 447 * regarding the unlinked set below to fail due to i/o error. On a
448 448 * nondebug system, this will result in the space being leaked.
449 449 */
450 450 void
451 451 zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx)
452 452 {
453 453 zfsvfs_t *zfsvfs = zp->z_zfsvfs;
454 454
455 455 ASSERT(zp->z_unlinked);
456 456 ASSERT(zp->z_links == 0);
457 457
458 458 VERIFY3U(0, ==,
459 459 zap_add_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
460 460 }
461 461
462 462 /*
463 463 * Clean up any znodes that had no links when we either crashed or
464 464 * (force) umounted the file system.
465 465 */
466 466 void
467 467 zfs_unlinked_drain(zfsvfs_t *zfsvfs)
468 468 {
469 469 zap_cursor_t zc;
470 470 zap_attribute_t zap;
471 471 dmu_object_info_t doi;
472 472 znode_t *zp;
473 473 int error;
474 474
475 475 /*
476 476 * Interate over the contents of the unlinked set.
477 477 */
478 478 for (zap_cursor_init(&zc, zfsvfs->z_os, zfsvfs->z_unlinkedobj);
479 479 zap_cursor_retrieve(&zc, &zap) == 0;
480 480 zap_cursor_advance(&zc)) {
481 481
482 482 /*
483 483 * See what kind of object we have in list
484 484 */
485 485
486 486 error = dmu_object_info(zfsvfs->z_os,
487 487 zap.za_first_integer, &doi);
488 488 if (error != 0)
489 489 continue;
490 490
491 491 ASSERT((doi.doi_type == DMU_OT_PLAIN_FILE_CONTENTS) ||
492 492 (doi.doi_type == DMU_OT_DIRECTORY_CONTENTS));
493 493 /*
494 494 * We need to re-mark these list entries for deletion,
495 495 * so we pull them back into core and set zp->z_unlinked.
496 496 */
497 497 error = zfs_zget(zfsvfs, zap.za_first_integer, &zp);
498 498
499 499 /*
500 500 * We may pick up znodes that are already marked for deletion.
501 501 * This could happen during the purge of an extended attribute
502 502 * directory. All we need to do is skip over them, since they
503 503 * are already in the system marked z_unlinked.
504 504 */
505 505 if (error != 0)
506 506 continue;
507 507
508 508 zp->z_unlinked = B_TRUE;
509 509 VN_RELE(ZTOV(zp));
510 510 }
511 511 zap_cursor_fini(&zc);
512 512 }
513 513
514 514 /*
515 515 * Delete the entire contents of a directory. Return a count
516 516 * of the number of entries that could not be deleted. If we encounter
517 517 * an error, return a count of at least one so that the directory stays
518 518 * in the unlinked set.
519 519 *
520 520 * NOTE: this function assumes that the directory is inactive,
521 521 * so there is no need to lock its entries before deletion.
522 522 * Also, it assumes the directory contents is *only* regular
523 523 * files.
524 524 */
525 525 static int
526 526 zfs_purgedir(znode_t *dzp)
527 527 {
528 528 zap_cursor_t zc;
529 529 zap_attribute_t zap;
530 530 znode_t *xzp;
531 531 dmu_tx_t *tx;
532 532 zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
533 533 zfs_dirlock_t dl;
534 534 int skipped = 0;
535 535 int error;
536 536
537 537 for (zap_cursor_init(&zc, zfsvfs->z_os, dzp->z_id);
538 538 (error = zap_cursor_retrieve(&zc, &zap)) == 0;
539 539 zap_cursor_advance(&zc)) {
540 540 error = zfs_zget(zfsvfs,
541 541 ZFS_DIRENT_OBJ(zap.za_first_integer), &xzp);
542 542 if (error) {
543 543 skipped += 1;
544 544 continue;
545 545 }
546 546
547 547 ASSERT((ZTOV(xzp)->v_type == VREG) ||
548 548 (ZTOV(xzp)->v_type == VLNK));
549 549
550 550 tx = dmu_tx_create(zfsvfs->z_os);
551 551 dmu_tx_hold_sa(tx, dzp->z_sa_hdl, B_FALSE);
552 552 dmu_tx_hold_zap(tx, dzp->z_id, FALSE, zap.za_name);
553 553 dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
554 554 dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
555 555 /* Is this really needed ? */
556 556 zfs_sa_upgrade_txholds(tx, xzp);
557 557 error = dmu_tx_assign(tx, TXG_WAIT);
558 558 if (error) {
559 559 dmu_tx_abort(tx);
560 560 VN_RELE(ZTOV(xzp));
561 561 skipped += 1;
562 562 continue;
563 563 }
564 564 bzero(&dl, sizeof (dl));
565 565 dl.dl_dzp = dzp;
566 566 dl.dl_name = zap.za_name;
567 567
568 568 error = zfs_link_destroy(&dl, xzp, tx, 0, NULL);
569 569 if (error)
570 570 skipped += 1;
571 571 dmu_tx_commit(tx);
572 572
573 573 VN_RELE(ZTOV(xzp));
574 574 }
575 575 zap_cursor_fini(&zc);
576 576 if (error != ENOENT)
577 577 skipped += 1;
578 578 return (skipped);
579 579 }
580 580
581 581 void
582 582 zfs_rmnode(znode_t *zp)
583 583 {
584 584 zfsvfs_t *zfsvfs = zp->z_zfsvfs;
585 585 objset_t *os = zfsvfs->z_os;
586 586 znode_t *xzp = NULL;
587 587 dmu_tx_t *tx;
588 588 uint64_t acl_obj;
589 589 uint64_t xattr_obj;
590 590 int error;
591 591
592 592 ASSERT(zp->z_links == 0);
593 593 ASSERT(ZTOV(zp)->v_count == 0);
594 594
595 595 /*
596 596 * If this is an attribute directory, purge its contents.
597 597 */
598 598 if (ZTOV(zp)->v_type == VDIR && (zp->z_pflags & ZFS_XATTR)) {
599 599 if (zfs_purgedir(zp) != 0) {
600 600 /*
601 601 * Not enough space to delete some xattrs.
602 602 * Leave it in the unlinked set.
603 603 */
604 604 zfs_znode_dmu_fini(zp);
605 605 zfs_znode_free(zp);
606 606 return;
607 607 }
608 608 }
609 609
610 610 /*
611 611 * Free up all the data in the file.
612 612 */
613 613 error = dmu_free_long_range(os, zp->z_id, 0, DMU_OBJECT_END);
614 614 if (error) {
615 615 /*
616 616 * Not enough space. Leave the file in the unlinked set.
617 617 */
618 618 zfs_znode_dmu_fini(zp);
619 619 zfs_znode_free(zp);
620 620 return;
621 621 }
622 622
623 623 /*
624 624 * If the file has extended attributes, we're going to unlink
625 625 * the xattr dir.
626 626 */
627 627 error = sa_lookup(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs),
628 628 &xattr_obj, sizeof (xattr_obj));
629 629 if (error == 0 && xattr_obj) {
630 630 error = zfs_zget(zfsvfs, xattr_obj, &xzp);
631 631 ASSERT(error == 0);
632 632 }
633 633
634 634 acl_obj = zfs_external_acl(zp);
635 635
636 636 /*
637 637 * Set up the final transaction.
638 638 */
639 639 tx = dmu_tx_create(os);
640 640 dmu_tx_hold_free(tx, zp->z_id, 0, DMU_OBJECT_END);
641 641 dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, FALSE, NULL);
642 642 if (xzp) {
643 643 dmu_tx_hold_zap(tx, zfsvfs->z_unlinkedobj, TRUE, NULL);
644 644 dmu_tx_hold_sa(tx, xzp->z_sa_hdl, B_FALSE);
645 645 }
646 646 if (acl_obj)
647 647 dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
648 648
649 649 zfs_sa_upgrade_txholds(tx, zp);
650 650 error = dmu_tx_assign(tx, TXG_WAIT);
651 651 if (error) {
652 652 /*
653 653 * Not enough space to delete the file. Leave it in the
654 654 * unlinked set, leaking it until the fs is remounted (at
655 655 * which point we'll call zfs_unlinked_drain() to process it).
656 656 */
657 657 dmu_tx_abort(tx);
658 658 zfs_znode_dmu_fini(zp);
659 659 zfs_znode_free(zp);
660 660 goto out;
661 661 }
662 662
663 663 if (xzp) {
664 664 ASSERT(error == 0);
665 665 mutex_enter(&xzp->z_lock);
666 666 xzp->z_unlinked = B_TRUE; /* mark xzp for deletion */
667 667 xzp->z_links = 0; /* no more links to it */
668 668 VERIFY(0 == sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zfsvfs),
669 669 &xzp->z_links, sizeof (xzp->z_links), tx));
670 670 mutex_exit(&xzp->z_lock);
671 671 zfs_unlinked_add(xzp, tx);
672 672 }
673 673
674 674 /* Remove this znode from the unlinked set */
675 675 VERIFY3U(0, ==,
676 676 zap_remove_int(zfsvfs->z_os, zfsvfs->z_unlinkedobj, zp->z_id, tx));
677 677
678 678 zfs_znode_delete(zp, tx);
679 679
680 680 dmu_tx_commit(tx);
681 681 out:
682 682 if (xzp)
683 683 VN_RELE(ZTOV(xzp));
684 684 }
685 685
686 686 static uint64_t
687 687 zfs_dirent(znode_t *zp, uint64_t mode)
688 688 {
689 689 uint64_t de = zp->z_id;
690 690
691 691 if (zp->z_zfsvfs->z_version >= ZPL_VERSION_DIRENT_TYPE)
692 692 de |= IFTODT(mode) << 60;
693 693 return (de);
694 694 }
695 695
696 696 /*
697 697 * Link zp into dl. Can only fail if zp has been unlinked.
698 698 */
699 699 int
700 700 zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag)
701 701 {
702 702 znode_t *dzp = dl->dl_dzp;
703 703 zfsvfs_t *zfsvfs = zp->z_zfsvfs;
704 704 vnode_t *vp = ZTOV(zp);
705 705 uint64_t value;
706 706 int zp_is_dir = (vp->v_type == VDIR);
707 707 sa_bulk_attr_t bulk[5];
708 708 uint64_t mtime[2], ctime[2];
709 709 int count = 0;
710 710 int error;
711 711
712 712 mutex_enter(&zp->z_lock);
713 713
714 714 if (!(flag & ZRENAMING)) {
715 715 if (zp->z_unlinked) { /* no new links to unlinked zp */
716 716 ASSERT(!(flag & (ZNEW | ZEXISTS)));
717 717 mutex_exit(&zp->z_lock);
718 718 return (SET_ERROR(ENOENT));
719 719 }
720 720 zp->z_links++;
721 721 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL,
722 722 &zp->z_links, sizeof (zp->z_links));
723 723
724 724 }
725 725 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
726 726 &dzp->z_id, sizeof (dzp->z_id));
727 727 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
728 728 &zp->z_pflags, sizeof (zp->z_pflags));
729 729
730 730 if (!(flag & ZNEW)) {
731 731 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
732 732 ctime, sizeof (ctime));
733 733 zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime,
734 734 ctime, B_TRUE);
735 735 }
736 736 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
737 737 ASSERT(error == 0);
738 738
739 739 mutex_exit(&zp->z_lock);
740 740
741 741 mutex_enter(&dzp->z_lock);
742 742 dzp->z_size++;
743 743 dzp->z_links += zp_is_dir;
744 744 count = 0;
745 745 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
746 746 &dzp->z_size, sizeof (dzp->z_size));
747 747 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs), NULL,
748 748 &dzp->z_links, sizeof (dzp->z_links));
749 749 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
750 750 mtime, sizeof (mtime));
751 751 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
752 752 ctime, sizeof (ctime));
753 753 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
754 754 &dzp->z_pflags, sizeof (dzp->z_pflags));
755 755 zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime, B_TRUE);
756 756 error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
757 757 ASSERT(error == 0);
758 758 mutex_exit(&dzp->z_lock);
759 759
760 760 value = zfs_dirent(zp, zp->z_mode);
761 761 error = zap_add(zp->z_zfsvfs->z_os, dzp->z_id, dl->dl_name,
762 762 8, 1, &value, tx);
763 763 ASSERT(error == 0);
764 764
765 765 dnlc_update(ZTOV(dzp), dl->dl_name, vp);
766 766
767 767 return (0);
768 768 }
769 769
770 770 static int
771 771 zfs_dropname(zfs_dirlock_t *dl, znode_t *zp, znode_t *dzp, dmu_tx_t *tx,
772 772 int flag)
773 773 {
774 774 int error;
775 775
776 776 if (zp->z_zfsvfs->z_norm) {
777 777 if (((zp->z_zfsvfs->z_case == ZFS_CASE_INSENSITIVE) &&
778 778 (flag & ZCIEXACT)) ||
779 779 ((zp->z_zfsvfs->z_case == ZFS_CASE_MIXED) &&
780 780 !(flag & ZCILOOK)))
781 781 error = zap_remove_norm(zp->z_zfsvfs->z_os,
782 782 dzp->z_id, dl->dl_name, MT_EXACT, tx);
783 783 else
784 784 error = zap_remove_norm(zp->z_zfsvfs->z_os,
785 785 dzp->z_id, dl->dl_name, MT_FIRST, tx);
786 786 } else {
787 787 error = zap_remove(zp->z_zfsvfs->z_os,
788 788 dzp->z_id, dl->dl_name, tx);
789 789 }
790 790
791 791 return (error);
792 792 }
793 793
794 794 /*
795 795 * Unlink zp from dl, and mark zp for deletion if this was the last link.
796 796 * Can fail if zp is a mount point (EBUSY) or a non-empty directory (EEXIST).
797 797 * If 'unlinkedp' is NULL, we put unlinked znodes on the unlinked list.
798 798 * If it's non-NULL, we use it to indicate whether the znode needs deletion,
799 799 * and it's the caller's job to do it.
800 800 */
801 801 int
802 802 zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag,
803 803 boolean_t *unlinkedp)
804 804 {
805 805 znode_t *dzp = dl->dl_dzp;
806 806 zfsvfs_t *zfsvfs = dzp->z_zfsvfs;
807 807 vnode_t *vp = ZTOV(zp);
808 808 int zp_is_dir = (vp->v_type == VDIR);
809 809 boolean_t unlinked = B_FALSE;
810 810 sa_bulk_attr_t bulk[5];
811 811 uint64_t mtime[2], ctime[2];
812 812 int count = 0;
813 813 int error;
814 814
815 815 dnlc_remove(ZTOV(dzp), dl->dl_name);
816 816
817 817 if (!(flag & ZRENAMING)) {
818 818 if (vn_vfswlock(vp)) /* prevent new mounts on zp */
819 819 return (SET_ERROR(EBUSY));
820 820
821 821 if (vn_ismntpt(vp)) { /* don't remove mount point */
822 822 vn_vfsunlock(vp);
823 823 return (SET_ERROR(EBUSY));
824 824 }
825 825
826 826 mutex_enter(&zp->z_lock);
827 827
828 828 if (zp_is_dir && !zfs_dirempty(zp)) {
829 829 mutex_exit(&zp->z_lock);
830 830 vn_vfsunlock(vp);
831 831 return (SET_ERROR(EEXIST));
832 832 }
833 833
834 834 /*
835 835 * If we get here, we are going to try to remove the object.
836 836 * First try removing the name from the directory; if that
837 837 * fails, return the error.
838 838 */
839 839 error = zfs_dropname(dl, zp, dzp, tx, flag);
840 840 if (error != 0) {
841 841 mutex_exit(&zp->z_lock);
842 842 vn_vfsunlock(vp);
843 843 return (error);
844 844 }
845 845
846 846 if (zp->z_links <= zp_is_dir) {
847 847 zfs_panic_recover("zfs: link count on %s is %u, "
848 848 "should be at least %u",
849 849 zp->z_vnode->v_path ? zp->z_vnode->v_path :
850 850 "<unknown>", (int)zp->z_links,
851 851 zp_is_dir + 1);
852 852 zp->z_links = zp_is_dir + 1;
853 853 }
854 854 if (--zp->z_links == zp_is_dir) {
855 855 zp->z_unlinked = B_TRUE;
856 856 zp->z_links = 0;
857 857 unlinked = B_TRUE;
858 858 } else {
859 859 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs),
860 860 NULL, &ctime, sizeof (ctime));
861 861 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
862 862 NULL, &zp->z_pflags, sizeof (zp->z_pflags));
863 863 zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime,
864 864 B_TRUE);
865 865 }
866 866 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
867 867 NULL, &zp->z_links, sizeof (zp->z_links));
868 868 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
869 869 count = 0;
870 870 ASSERT(error == 0);
871 871 mutex_exit(&zp->z_lock);
872 872 vn_vfsunlock(vp);
873 873 } else {
874 874 error = zfs_dropname(dl, zp, dzp, tx, flag);
875 875 if (error != 0)
876 876 return (error);
877 877 }
878 878
879 879 mutex_enter(&dzp->z_lock);
880 880 dzp->z_size--; /* one dirent removed */
881 881 dzp->z_links -= zp_is_dir; /* ".." link from zp */
882 882 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zfsvfs),
883 883 NULL, &dzp->z_links, sizeof (dzp->z_links));
884 884 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs),
885 885 NULL, &dzp->z_size, sizeof (dzp->z_size));
886 886 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs),
887 887 NULL, ctime, sizeof (ctime));
888 888 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs),
889 889 NULL, mtime, sizeof (mtime));
890 890 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs),
891 891 NULL, &dzp->z_pflags, sizeof (dzp->z_pflags));
892 892 zfs_tstamp_update_setup(dzp, CONTENT_MODIFIED, mtime, ctime, B_TRUE);
893 893 error = sa_bulk_update(dzp->z_sa_hdl, bulk, count, tx);
894 894 ASSERT(error == 0);
895 895 mutex_exit(&dzp->z_lock);
896 896
897 897 if (unlinkedp != NULL)
898 898 *unlinkedp = unlinked;
899 899 else if (unlinked)
900 900 zfs_unlinked_add(zp, tx);
901 901
902 902 return (0);
903 903 }
904 904
905 905 /*
906 906 * Indicate whether the directory is empty. Works with or without z_lock
907 907 * held, but can only be consider a hint in the latter case. Returns true
908 908 * if only "." and ".." remain and there's no work in progress.
909 909 */
910 910 boolean_t
911 911 zfs_dirempty(znode_t *dzp)
912 912 {
913 913 return (dzp->z_size == 2 && dzp->z_dirlocks == 0);
914 914 }
915 915
916 916 int
917 917 zfs_make_xattrdir(znode_t *zp, vattr_t *vap, vnode_t **xvpp, cred_t *cr)
918 918 {
919 919 zfsvfs_t *zfsvfs = zp->z_zfsvfs;
920 920 znode_t *xzp;
921 921 dmu_tx_t *tx;
922 922 int error;
923 923 zfs_acl_ids_t acl_ids;
924 924 boolean_t fuid_dirtied;
925 925 uint64_t parent;
926 926
927 927 *xvpp = NULL;
928 928
929 929 if (error = zfs_zaccess(zp, ACE_WRITE_NAMED_ATTRS, 0, B_FALSE, cr))
|
↓ open down ↓ |
929 lines elided |
↑ open up ↑ |
930 930 return (error);
931 931
932 932 if ((error = zfs_acl_ids_create(zp, IS_XATTR, vap, cr, NULL,
933 933 &acl_ids)) != 0)
934 934 return (error);
935 935 if (zfs_acl_ids_overquota(zfsvfs, &acl_ids)) {
936 936 zfs_acl_ids_free(&acl_ids);
937 937 return (SET_ERROR(EDQUOT));
938 938 }
939 939
940 -top:
941 940 tx = dmu_tx_create(zfsvfs->z_os);
942 941 dmu_tx_hold_sa_create(tx, acl_ids.z_aclp->z_acl_bytes +
943 942 ZFS_SA_BASE_ATTR_SIZE);
944 943 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_TRUE);
945 944 dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
946 945 fuid_dirtied = zfsvfs->z_fuid_dirty;
947 946 if (fuid_dirtied)
948 947 zfs_fuid_txhold(zfsvfs, tx);
949 - error = dmu_tx_assign(tx, TXG_NOWAIT);
948 + error = dmu_tx_assign(tx, TXG_WAIT);
950 949 if (error) {
951 - if (error == ERESTART) {
952 - dmu_tx_wait(tx);
953 - dmu_tx_abort(tx);
954 - goto top;
955 - }
956 950 zfs_acl_ids_free(&acl_ids);
957 951 dmu_tx_abort(tx);
958 952 return (error);
959 953 }
960 954 zfs_mknode(zp, vap, tx, cr, IS_XATTR, &xzp, &acl_ids);
961 955
962 956 if (fuid_dirtied)
963 957 zfs_fuid_sync(zfsvfs, tx);
964 958
965 959 #ifdef DEBUG
966 960 error = sa_lookup(xzp->z_sa_hdl, SA_ZPL_PARENT(zfsvfs),
967 961 &parent, sizeof (parent));
968 962 ASSERT(error == 0 && parent == zp->z_id);
969 963 #endif
970 964
971 965 VERIFY(0 == sa_update(zp->z_sa_hdl, SA_ZPL_XATTR(zfsvfs), &xzp->z_id,
972 966 sizeof (xzp->z_id), tx));
973 967
974 968 (void) zfs_log_create(zfsvfs->z_log, tx, TX_MKXATTR, zp,
975 969 xzp, "", NULL, acl_ids.z_fuidp, vap);
976 970
977 971 zfs_acl_ids_free(&acl_ids);
978 972 dmu_tx_commit(tx);
979 973
980 974 *xvpp = ZTOV(xzp);
981 975
982 976 return (0);
983 977 }
984 978
985 979 /*
986 980 * Return a znode for the extended attribute directory for zp.
987 981 * ** If the directory does not already exist, it is created **
988 982 *
989 983 * IN: zp - znode to obtain attribute directory from
990 984 * cr - credentials of caller
991 985 * flags - flags from the VOP_LOOKUP call
992 986 *
993 987 * OUT: xzpp - pointer to extended attribute znode
994 988 *
995 989 * RETURN: 0 on success
996 990 * error number on failure
997 991 */
998 992 int
999 993 zfs_get_xattrdir(znode_t *zp, vnode_t **xvpp, cred_t *cr, int flags)
1000 994 {
1001 995 zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1002 996 znode_t *xzp;
1003 997 zfs_dirlock_t *dl;
1004 998 vattr_t va;
1005 999 int error;
1006 1000 top:
1007 1001 error = zfs_dirent_lock(&dl, zp, "", &xzp, ZXATTR, NULL, NULL);
1008 1002 if (error)
1009 1003 return (error);
1010 1004
1011 1005 if (xzp != NULL) {
1012 1006 *xvpp = ZTOV(xzp);
1013 1007 zfs_dirent_unlock(dl);
1014 1008 return (0);
1015 1009 }
1016 1010
1017 1011
1018 1012 if (!(flags & CREATE_XATTR_DIR)) {
1019 1013 zfs_dirent_unlock(dl);
1020 1014 return (SET_ERROR(ENOENT));
1021 1015 }
1022 1016
1023 1017 if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) {
1024 1018 zfs_dirent_unlock(dl);
1025 1019 return (SET_ERROR(EROFS));
1026 1020 }
1027 1021
1028 1022 /*
1029 1023 * The ability to 'create' files in an attribute
1030 1024 * directory comes from the write_xattr permission on the base file.
1031 1025 *
1032 1026 * The ability to 'search' an attribute directory requires
1033 1027 * read_xattr permission on the base file.
1034 1028 *
1035 1029 * Once in a directory the ability to read/write attributes
1036 1030 * is controlled by the permissions on the attribute file.
1037 1031 */
1038 1032 va.va_mask = AT_TYPE | AT_MODE | AT_UID | AT_GID;
1039 1033 va.va_type = VDIR;
1040 1034 va.va_mode = S_IFDIR | S_ISVTX | 0777;
1041 1035 zfs_fuid_map_ids(zp, cr, &va.va_uid, &va.va_gid);
1042 1036
1043 1037 error = zfs_make_xattrdir(zp, &va, xvpp, cr);
1044 1038 zfs_dirent_unlock(dl);
1045 1039
1046 1040 if (error == ERESTART) {
1047 1041 /* NB: we already did dmu_tx_wait() if necessary */
1048 1042 goto top;
1049 1043 }
1050 1044
1051 1045 return (error);
1052 1046 }
1053 1047
1054 1048 /*
1055 1049 * Decide whether it is okay to remove within a sticky directory.
1056 1050 *
1057 1051 * In sticky directories, write access is not sufficient;
1058 1052 * you can remove entries from a directory only if:
1059 1053 *
1060 1054 * you own the directory,
1061 1055 * you own the entry,
1062 1056 * the entry is a plain file and you have write access,
1063 1057 * or you are privileged (checked in secpolicy...).
1064 1058 *
1065 1059 * The function returns 0 if remove access is granted.
1066 1060 */
1067 1061 int
1068 1062 zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr)
1069 1063 {
1070 1064 uid_t uid;
1071 1065 uid_t downer;
1072 1066 uid_t fowner;
1073 1067 zfsvfs_t *zfsvfs = zdp->z_zfsvfs;
1074 1068
1075 1069 if (zdp->z_zfsvfs->z_replay)
1076 1070 return (0);
1077 1071
1078 1072 if ((zdp->z_mode & S_ISVTX) == 0)
1079 1073 return (0);
1080 1074
1081 1075 downer = zfs_fuid_map_id(zfsvfs, zdp->z_uid, cr, ZFS_OWNER);
1082 1076 fowner = zfs_fuid_map_id(zfsvfs, zp->z_uid, cr, ZFS_OWNER);
1083 1077
1084 1078 if ((uid = crgetuid(cr)) == downer || uid == fowner ||
1085 1079 (ZTOV(zp)->v_type == VREG &&
1086 1080 zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr) == 0))
1087 1081 return (0);
1088 1082 else
1089 1083 return (secpolicy_vnode_remove(cr));
1090 1084 }
|
↓ open down ↓ |
125 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX