Print this page
4045 zfs write throttle & i/o scheduler performance work
Reviewed by: George Wilson <george.wilson@delphix.com>
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: Christopher Siden <christopher.siden@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/dmu.c
+++ new/usr/src/uts/common/fs/zfs/dmu.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright (c) 2013 by Delphix. All rights reserved.
24 24 */
25 25
26 26 /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
27 27
28 28 #include <sys/dmu.h>
29 29 #include <sys/dmu_impl.h>
30 30 #include <sys/dmu_tx.h>
31 31 #include <sys/dbuf.h>
32 32 #include <sys/dnode.h>
33 33 #include <sys/zfs_context.h>
34 34 #include <sys/dmu_objset.h>
35 35 #include <sys/dmu_traverse.h>
36 36 #include <sys/dsl_dataset.h>
37 37 #include <sys/dsl_dir.h>
38 38 #include <sys/dsl_pool.h>
39 39 #include <sys/dsl_synctask.h>
40 40 #include <sys/dsl_prop.h>
41 41 #include <sys/dmu_zfetch.h>
42 42 #include <sys/zfs_ioctl.h>
43 43 #include <sys/zap.h>
44 44 #include <sys/zio_checksum.h>
45 45 #include <sys/zio_compress.h>
46 46 #include <sys/sa.h>
47 47 #ifdef _KERNEL
48 48 #include <sys/vmsystm.h>
49 49 #include <sys/zfs_znode.h>
50 50 #endif
51 51
52 52 /*
53 53 * Enable/disable nopwrite feature.
54 54 */
55 55 int zfs_nopwrite_enabled = 1;
56 56
57 57 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
58 58 { DMU_BSWAP_UINT8, TRUE, "unallocated" },
59 59 { DMU_BSWAP_ZAP, TRUE, "object directory" },
60 60 { DMU_BSWAP_UINT64, TRUE, "object array" },
61 61 { DMU_BSWAP_UINT8, TRUE, "packed nvlist" },
62 62 { DMU_BSWAP_UINT64, TRUE, "packed nvlist size" },
63 63 { DMU_BSWAP_UINT64, TRUE, "bpobj" },
64 64 { DMU_BSWAP_UINT64, TRUE, "bpobj header" },
65 65 { DMU_BSWAP_UINT64, TRUE, "SPA space map header" },
66 66 { DMU_BSWAP_UINT64, TRUE, "SPA space map" },
67 67 { DMU_BSWAP_UINT64, TRUE, "ZIL intent log" },
68 68 { DMU_BSWAP_DNODE, TRUE, "DMU dnode" },
69 69 { DMU_BSWAP_OBJSET, TRUE, "DMU objset" },
70 70 { DMU_BSWAP_UINT64, TRUE, "DSL directory" },
71 71 { DMU_BSWAP_ZAP, TRUE, "DSL directory child map"},
72 72 { DMU_BSWAP_ZAP, TRUE, "DSL dataset snap map" },
73 73 { DMU_BSWAP_ZAP, TRUE, "DSL props" },
74 74 { DMU_BSWAP_UINT64, TRUE, "DSL dataset" },
75 75 { DMU_BSWAP_ZNODE, TRUE, "ZFS znode" },
76 76 { DMU_BSWAP_OLDACL, TRUE, "ZFS V0 ACL" },
77 77 { DMU_BSWAP_UINT8, FALSE, "ZFS plain file" },
78 78 { DMU_BSWAP_ZAP, TRUE, "ZFS directory" },
79 79 { DMU_BSWAP_ZAP, TRUE, "ZFS master node" },
80 80 { DMU_BSWAP_ZAP, TRUE, "ZFS delete queue" },
81 81 { DMU_BSWAP_UINT8, FALSE, "zvol object" },
82 82 { DMU_BSWAP_ZAP, TRUE, "zvol prop" },
83 83 { DMU_BSWAP_UINT8, FALSE, "other uint8[]" },
84 84 { DMU_BSWAP_UINT64, FALSE, "other uint64[]" },
85 85 { DMU_BSWAP_ZAP, TRUE, "other ZAP" },
86 86 { DMU_BSWAP_ZAP, TRUE, "persistent error log" },
87 87 { DMU_BSWAP_UINT8, TRUE, "SPA history" },
88 88 { DMU_BSWAP_UINT64, TRUE, "SPA history offsets" },
89 89 { DMU_BSWAP_ZAP, TRUE, "Pool properties" },
90 90 { DMU_BSWAP_ZAP, TRUE, "DSL permissions" },
91 91 { DMU_BSWAP_ACL, TRUE, "ZFS ACL" },
92 92 { DMU_BSWAP_UINT8, TRUE, "ZFS SYSACL" },
93 93 { DMU_BSWAP_UINT8, TRUE, "FUID table" },
94 94 { DMU_BSWAP_UINT64, TRUE, "FUID table size" },
95 95 { DMU_BSWAP_ZAP, TRUE, "DSL dataset next clones"},
96 96 { DMU_BSWAP_ZAP, TRUE, "scan work queue" },
97 97 { DMU_BSWAP_ZAP, TRUE, "ZFS user/group used" },
98 98 { DMU_BSWAP_ZAP, TRUE, "ZFS user/group quota" },
99 99 { DMU_BSWAP_ZAP, TRUE, "snapshot refcount tags"},
100 100 { DMU_BSWAP_ZAP, TRUE, "DDT ZAP algorithm" },
101 101 { DMU_BSWAP_ZAP, TRUE, "DDT statistics" },
102 102 { DMU_BSWAP_UINT8, TRUE, "System attributes" },
103 103 { DMU_BSWAP_ZAP, TRUE, "SA master node" },
104 104 { DMU_BSWAP_ZAP, TRUE, "SA attr registration" },
105 105 { DMU_BSWAP_ZAP, TRUE, "SA attr layouts" },
106 106 { DMU_BSWAP_ZAP, TRUE, "scan translations" },
107 107 { DMU_BSWAP_UINT8, FALSE, "deduplicated block" },
108 108 { DMU_BSWAP_ZAP, TRUE, "DSL deadlist map" },
109 109 { DMU_BSWAP_UINT64, TRUE, "DSL deadlist map hdr" },
110 110 { DMU_BSWAP_ZAP, TRUE, "DSL dir clones" },
111 111 { DMU_BSWAP_UINT64, TRUE, "bpobj subobj" }
112 112 };
113 113
114 114 const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
115 115 { byteswap_uint8_array, "uint8" },
116 116 { byteswap_uint16_array, "uint16" },
117 117 { byteswap_uint32_array, "uint32" },
118 118 { byteswap_uint64_array, "uint64" },
119 119 { zap_byteswap, "zap" },
120 120 { dnode_buf_byteswap, "dnode" },
121 121 { dmu_objset_byteswap, "objset" },
122 122 { zfs_znode_byteswap, "znode" },
123 123 { zfs_oldacl_byteswap, "oldacl" },
124 124 { zfs_acl_byteswap, "acl" }
125 125 };
126 126
127 127 int
128 128 dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
129 129 void *tag, dmu_buf_t **dbp, int flags)
130 130 {
131 131 dnode_t *dn;
132 132 uint64_t blkid;
133 133 dmu_buf_impl_t *db;
134 134 int err;
135 135 int db_flags = DB_RF_CANFAIL;
136 136
137 137 if (flags & DMU_READ_NO_PREFETCH)
138 138 db_flags |= DB_RF_NOPREFETCH;
139 139
140 140 err = dnode_hold(os, object, FTAG, &dn);
141 141 if (err)
142 142 return (err);
143 143 blkid = dbuf_whichblock(dn, offset);
144 144 rw_enter(&dn->dn_struct_rwlock, RW_READER);
145 145 db = dbuf_hold(dn, blkid, tag);
146 146 rw_exit(&dn->dn_struct_rwlock);
147 147 if (db == NULL) {
148 148 err = SET_ERROR(EIO);
149 149 } else {
150 150 err = dbuf_read(db, NULL, db_flags);
151 151 if (err) {
152 152 dbuf_rele(db, tag);
153 153 db = NULL;
154 154 }
155 155 }
156 156
157 157 dnode_rele(dn, FTAG);
158 158 *dbp = &db->db; /* NULL db plus first field offset is NULL */
159 159 return (err);
160 160 }
161 161
162 162 int
163 163 dmu_bonus_max(void)
164 164 {
165 165 return (DN_MAX_BONUSLEN);
166 166 }
167 167
168 168 int
169 169 dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
170 170 {
171 171 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
172 172 dnode_t *dn;
173 173 int error;
174 174
175 175 DB_DNODE_ENTER(db);
176 176 dn = DB_DNODE(db);
177 177
178 178 if (dn->dn_bonus != db) {
179 179 error = SET_ERROR(EINVAL);
180 180 } else if (newsize < 0 || newsize > db_fake->db_size) {
181 181 error = SET_ERROR(EINVAL);
182 182 } else {
183 183 dnode_setbonuslen(dn, newsize, tx);
184 184 error = 0;
185 185 }
186 186
187 187 DB_DNODE_EXIT(db);
188 188 return (error);
189 189 }
190 190
191 191 int
192 192 dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
193 193 {
194 194 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
195 195 dnode_t *dn;
196 196 int error;
197 197
198 198 DB_DNODE_ENTER(db);
199 199 dn = DB_DNODE(db);
200 200
201 201 if (!DMU_OT_IS_VALID(type)) {
202 202 error = SET_ERROR(EINVAL);
203 203 } else if (dn->dn_bonus != db) {
204 204 error = SET_ERROR(EINVAL);
205 205 } else {
206 206 dnode_setbonus_type(dn, type, tx);
207 207 error = 0;
208 208 }
209 209
210 210 DB_DNODE_EXIT(db);
211 211 return (error);
212 212 }
213 213
214 214 dmu_object_type_t
215 215 dmu_get_bonustype(dmu_buf_t *db_fake)
216 216 {
217 217 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
218 218 dnode_t *dn;
219 219 dmu_object_type_t type;
220 220
221 221 DB_DNODE_ENTER(db);
222 222 dn = DB_DNODE(db);
223 223 type = dn->dn_bonustype;
224 224 DB_DNODE_EXIT(db);
225 225
226 226 return (type);
227 227 }
228 228
229 229 int
230 230 dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
231 231 {
232 232 dnode_t *dn;
233 233 int error;
234 234
235 235 error = dnode_hold(os, object, FTAG, &dn);
236 236 dbuf_rm_spill(dn, tx);
237 237 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
238 238 dnode_rm_spill(dn, tx);
239 239 rw_exit(&dn->dn_struct_rwlock);
240 240 dnode_rele(dn, FTAG);
241 241 return (error);
242 242 }
243 243
244 244 /*
245 245 * returns ENOENT, EIO, or 0.
246 246 */
247 247 int
248 248 dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
249 249 {
250 250 dnode_t *dn;
251 251 dmu_buf_impl_t *db;
252 252 int error;
253 253
254 254 error = dnode_hold(os, object, FTAG, &dn);
255 255 if (error)
256 256 return (error);
257 257
258 258 rw_enter(&dn->dn_struct_rwlock, RW_READER);
259 259 if (dn->dn_bonus == NULL) {
260 260 rw_exit(&dn->dn_struct_rwlock);
261 261 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
262 262 if (dn->dn_bonus == NULL)
263 263 dbuf_create_bonus(dn);
264 264 }
265 265 db = dn->dn_bonus;
266 266
267 267 /* as long as the bonus buf is held, the dnode will be held */
268 268 if (refcount_add(&db->db_holds, tag) == 1) {
269 269 VERIFY(dnode_add_ref(dn, db));
270 270 (void) atomic_inc_32_nv(&dn->dn_dbufs_count);
271 271 }
272 272
273 273 /*
274 274 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
275 275 * hold and incrementing the dbuf count to ensure that dnode_move() sees
276 276 * a dnode hold for every dbuf.
277 277 */
278 278 rw_exit(&dn->dn_struct_rwlock);
279 279
280 280 dnode_rele(dn, FTAG);
281 281
282 282 VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH));
283 283
284 284 *dbp = &db->db;
285 285 return (0);
286 286 }
287 287
288 288 /*
289 289 * returns ENOENT, EIO, or 0.
290 290 *
291 291 * This interface will allocate a blank spill dbuf when a spill blk
292 292 * doesn't already exist on the dnode.
293 293 *
294 294 * if you only want to find an already existing spill db, then
295 295 * dmu_spill_hold_existing() should be used.
296 296 */
297 297 int
298 298 dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
299 299 {
300 300 dmu_buf_impl_t *db = NULL;
301 301 int err;
302 302
303 303 if ((flags & DB_RF_HAVESTRUCT) == 0)
304 304 rw_enter(&dn->dn_struct_rwlock, RW_READER);
305 305
306 306 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
307 307
308 308 if ((flags & DB_RF_HAVESTRUCT) == 0)
309 309 rw_exit(&dn->dn_struct_rwlock);
310 310
311 311 ASSERT(db != NULL);
312 312 err = dbuf_read(db, NULL, flags);
313 313 if (err == 0)
314 314 *dbp = &db->db;
315 315 else
316 316 dbuf_rele(db, tag);
317 317 return (err);
318 318 }
319 319
320 320 int
321 321 dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
322 322 {
323 323 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
324 324 dnode_t *dn;
325 325 int err;
326 326
327 327 DB_DNODE_ENTER(db);
328 328 dn = DB_DNODE(db);
329 329
330 330 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
331 331 err = SET_ERROR(EINVAL);
332 332 } else {
333 333 rw_enter(&dn->dn_struct_rwlock, RW_READER);
334 334
335 335 if (!dn->dn_have_spill) {
336 336 err = SET_ERROR(ENOENT);
337 337 } else {
338 338 err = dmu_spill_hold_by_dnode(dn,
339 339 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
340 340 }
341 341
342 342 rw_exit(&dn->dn_struct_rwlock);
343 343 }
344 344
345 345 DB_DNODE_EXIT(db);
346 346 return (err);
347 347 }
348 348
349 349 int
350 350 dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
351 351 {
352 352 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
353 353 dnode_t *dn;
354 354 int err;
355 355
356 356 DB_DNODE_ENTER(db);
357 357 dn = DB_DNODE(db);
358 358 err = dmu_spill_hold_by_dnode(dn, DB_RF_CANFAIL, tag, dbp);
359 359 DB_DNODE_EXIT(db);
360 360
361 361 return (err);
362 362 }
363 363
|
↓ open down ↓ |
363 lines elided |
↑ open up ↑ |
364 364 /*
365 365 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
366 366 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
367 367 * and can induce severe lock contention when writing to several files
368 368 * whose dnodes are in the same block.
369 369 */
370 370 static int
371 371 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
372 372 int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
373 373 {
374 - dsl_pool_t *dp = NULL;
375 374 dmu_buf_t **dbp;
376 375 uint64_t blkid, nblks, i;
377 376 uint32_t dbuf_flags;
378 377 int err;
379 378 zio_t *zio;
380 - hrtime_t start;
381 379
382 380 ASSERT(length <= DMU_MAX_ACCESS);
383 381
384 382 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT;
385 383 if (flags & DMU_READ_NO_PREFETCH || length > zfetch_array_rd_sz)
386 384 dbuf_flags |= DB_RF_NOPREFETCH;
387 385
388 386 rw_enter(&dn->dn_struct_rwlock, RW_READER);
389 387 if (dn->dn_datablkshift) {
390 388 int blkshift = dn->dn_datablkshift;
391 389 nblks = (P2ROUNDUP(offset+length, 1ULL<<blkshift) -
392 390 P2ALIGN(offset, 1ULL<<blkshift)) >> blkshift;
393 391 } else {
394 392 if (offset + length > dn->dn_datablksz) {
395 393 zfs_panic_recover("zfs: accessing past end of object "
396 394 "%llx/%llx (size=%u access=%llu+%llu)",
397 395 (longlong_t)dn->dn_objset->
|
↓ open down ↓ |
7 lines elided |
↑ open up ↑ |
398 396 os_dsl_dataset->ds_object,
399 397 (longlong_t)dn->dn_object, dn->dn_datablksz,
400 398 (longlong_t)offset, (longlong_t)length);
401 399 rw_exit(&dn->dn_struct_rwlock);
402 400 return (SET_ERROR(EIO));
403 401 }
404 402 nblks = 1;
405 403 }
406 404 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
407 405
408 - if (dn->dn_objset->os_dsl_dataset)
409 - dp = dn->dn_objset->os_dsl_dataset->ds_dir->dd_pool;
410 - start = gethrtime();
411 406 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
412 407 blkid = dbuf_whichblock(dn, offset);
413 408 for (i = 0; i < nblks; i++) {
414 409 dmu_buf_impl_t *db = dbuf_hold(dn, blkid+i, tag);
415 410 if (db == NULL) {
416 411 rw_exit(&dn->dn_struct_rwlock);
417 412 dmu_buf_rele_array(dbp, nblks, tag);
418 413 zio_nowait(zio);
419 414 return (SET_ERROR(EIO));
420 415 }
421 416 /* initiate async i/o */
422 417 if (read) {
423 418 (void) dbuf_read(db, zio, dbuf_flags);
424 419 }
425 420 dbp[i] = &db->db;
426 421 }
427 422 rw_exit(&dn->dn_struct_rwlock);
428 423
429 424 /* wait for async i/o */
430 425 err = zio_wait(zio);
431 - /* track read overhead when we are in sync context */
432 - if (dp && dsl_pool_sync_context(dp))
433 - dp->dp_read_overhead += gethrtime() - start;
434 426 if (err) {
435 427 dmu_buf_rele_array(dbp, nblks, tag);
436 428 return (err);
437 429 }
438 430
439 431 /* wait for other io to complete */
440 432 if (read) {
441 433 for (i = 0; i < nblks; i++) {
442 434 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
443 435 mutex_enter(&db->db_mtx);
444 436 while (db->db_state == DB_READ ||
445 437 db->db_state == DB_FILL)
446 438 cv_wait(&db->db_changed, &db->db_mtx);
447 439 if (db->db_state == DB_UNCACHED)
448 440 err = SET_ERROR(EIO);
449 441 mutex_exit(&db->db_mtx);
450 442 if (err) {
451 443 dmu_buf_rele_array(dbp, nblks, tag);
452 444 return (err);
453 445 }
454 446 }
455 447 }
456 448
457 449 *numbufsp = nblks;
458 450 *dbpp = dbp;
459 451 return (0);
460 452 }
461 453
462 454 static int
463 455 dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
464 456 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
465 457 {
466 458 dnode_t *dn;
467 459 int err;
468 460
469 461 err = dnode_hold(os, object, FTAG, &dn);
470 462 if (err)
471 463 return (err);
472 464
473 465 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
474 466 numbufsp, dbpp, DMU_READ_PREFETCH);
475 467
476 468 dnode_rele(dn, FTAG);
477 469
478 470 return (err);
479 471 }
480 472
481 473 int
482 474 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
483 475 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
484 476 {
485 477 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
486 478 dnode_t *dn;
487 479 int err;
488 480
489 481 DB_DNODE_ENTER(db);
490 482 dn = DB_DNODE(db);
491 483 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
492 484 numbufsp, dbpp, DMU_READ_PREFETCH);
493 485 DB_DNODE_EXIT(db);
494 486
495 487 return (err);
496 488 }
497 489
498 490 void
499 491 dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
500 492 {
501 493 int i;
502 494 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
503 495
504 496 if (numbufs == 0)
|
↓ open down ↓ |
61 lines elided |
↑ open up ↑ |
505 497 return;
506 498
507 499 for (i = 0; i < numbufs; i++) {
508 500 if (dbp[i])
509 501 dbuf_rele(dbp[i], tag);
510 502 }
511 503
512 504 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
513 505 }
514 506
507 +/*
508 + * Issue prefetch i/os for the given blocks.
509 + *
510 + * Note: The assumption is that we *know* these blocks will be needed
511 + * almost immediately. Therefore, the prefetch i/os will be issued at
512 + * ZIO_PRIORITY_SYNC_READ
513 + *
514 + * Note: indirect blocks and other metadata will be read synchronously,
515 + * causing this function to block if they are not already cached.
516 + */
515 517 void
516 518 dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset, uint64_t len)
517 519 {
518 520 dnode_t *dn;
519 521 uint64_t blkid;
520 - int nblks, i, err;
522 + int nblks, err;
521 523
522 524 if (zfs_prefetch_disable)
523 525 return;
524 526
525 527 if (len == 0) { /* they're interested in the bonus buffer */
526 528 dn = DMU_META_DNODE(os);
527 529
528 530 if (object == 0 || object >= DN_MAX_OBJECT)
529 531 return;
530 532
531 533 rw_enter(&dn->dn_struct_rwlock, RW_READER);
532 534 blkid = dbuf_whichblock(dn, object * sizeof (dnode_phys_t));
533 - dbuf_prefetch(dn, blkid);
535 + dbuf_prefetch(dn, blkid, ZIO_PRIORITY_SYNC_READ);
534 536 rw_exit(&dn->dn_struct_rwlock);
535 537 return;
536 538 }
537 539
538 540 /*
539 541 * XXX - Note, if the dnode for the requested object is not
540 542 * already cached, we will do a *synchronous* read in the
541 543 * dnode_hold() call. The same is true for any indirects.
542 544 */
543 545 err = dnode_hold(os, object, FTAG, &dn);
544 546 if (err != 0)
545 547 return;
546 548
547 549 rw_enter(&dn->dn_struct_rwlock, RW_READER);
548 550 if (dn->dn_datablkshift) {
549 551 int blkshift = dn->dn_datablkshift;
550 - nblks = (P2ROUNDUP(offset+len, 1<<blkshift) -
551 - P2ALIGN(offset, 1<<blkshift)) >> blkshift;
552 + nblks = (P2ROUNDUP(offset + len, 1 << blkshift) -
553 + P2ALIGN(offset, 1 << blkshift)) >> blkshift;
552 554 } else {
553 555 nblks = (offset < dn->dn_datablksz);
554 556 }
555 557
556 558 if (nblks != 0) {
557 559 blkid = dbuf_whichblock(dn, offset);
558 - for (i = 0; i < nblks; i++)
559 - dbuf_prefetch(dn, blkid+i);
560 + for (int i = 0; i < nblks; i++)
561 + dbuf_prefetch(dn, blkid + i, ZIO_PRIORITY_SYNC_READ);
560 562 }
561 563
562 564 rw_exit(&dn->dn_struct_rwlock);
563 565
564 566 dnode_rele(dn, FTAG);
565 567 }
566 568
567 569 /*
568 570 * Get the next "chunk" of file data to free. We traverse the file from
569 571 * the end so that the file gets shorter over time (if we crashes in the
570 572 * middle, this will leave us in a better state). We find allocated file
571 573 * data by simply searching the allocated level 1 indirects.
572 574 *
573 575 * On input, *start should be the first offset that does not need to be
574 576 * freed (e.g. "offset + length"). On return, *start will be the first
575 577 * offset that should be freed.
576 578 */
577 579 static int
578 580 get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum)
579 581 {
580 582 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
581 583 /* bytes of data covered by a level-1 indirect block */
582 584 uint64_t iblkrange =
583 585 dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
584 586
585 587 ASSERT3U(minimum, <=, *start);
586 588
587 589 if (*start - minimum <= iblkrange * maxblks) {
588 590 *start = minimum;
589 591 return (0);
590 592 }
591 593 ASSERT(ISP2(iblkrange));
592 594
593 595 for (uint64_t blks = 0; *start > minimum && blks < maxblks; blks++) {
594 596 int err;
595 597
596 598 /*
597 599 * dnode_next_offset(BACKWARDS) will find an allocated L1
598 600 * indirect block at or before the input offset. We must
599 601 * decrement *start so that it is at the end of the region
600 602 * to search.
601 603 */
602 604 (*start)--;
603 605 err = dnode_next_offset(dn,
604 606 DNODE_FIND_BACKWARDS, start, 2, 1, 0);
605 607
606 608 /* if there are no indirect blocks before start, we are done */
607 609 if (err == ESRCH) {
608 610 *start = minimum;
609 611 break;
610 612 } else if (err != 0) {
611 613 return (err);
612 614 }
613 615
614 616 /* set start to the beginning of this L1 indirect */
615 617 *start = P2ALIGN(*start, iblkrange);
616 618 }
617 619 if (*start < minimum)
618 620 *start = minimum;
619 621 return (0);
620 622 }
621 623
622 624 static int
623 625 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
624 626 uint64_t length)
625 627 {
626 628 uint64_t object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
627 629 int err;
628 630
629 631 if (offset >= object_size)
630 632 return (0);
631 633
632 634 if (length == DMU_OBJECT_END || offset + length > object_size)
633 635 length = object_size - offset;
634 636
635 637 while (length != 0) {
636 638 uint64_t chunk_end, chunk_begin;
637 639
638 640 chunk_end = chunk_begin = offset + length;
639 641
640 642 /* move chunk_begin backwards to the beginning of this chunk */
641 643 err = get_next_chunk(dn, &chunk_begin, offset);
642 644 if (err)
643 645 return (err);
644 646 ASSERT3U(chunk_begin, >=, offset);
645 647 ASSERT3U(chunk_begin, <=, chunk_end);
646 648
647 649 dmu_tx_t *tx = dmu_tx_create(os);
648 650 dmu_tx_hold_free(tx, dn->dn_object,
649 651 chunk_begin, chunk_end - chunk_begin);
650 652 err = dmu_tx_assign(tx, TXG_WAIT);
651 653 if (err) {
652 654 dmu_tx_abort(tx);
653 655 return (err);
654 656 }
655 657 dnode_free_range(dn, chunk_begin, chunk_end - chunk_begin, tx);
656 658 dmu_tx_commit(tx);
657 659
658 660 length -= chunk_end - chunk_begin;
659 661 }
660 662 return (0);
661 663 }
662 664
663 665 int
664 666 dmu_free_long_range(objset_t *os, uint64_t object,
665 667 uint64_t offset, uint64_t length)
666 668 {
667 669 dnode_t *dn;
668 670 int err;
669 671
670 672 err = dnode_hold(os, object, FTAG, &dn);
671 673 if (err != 0)
672 674 return (err);
673 675 err = dmu_free_long_range_impl(os, dn, offset, length);
674 676 dnode_rele(dn, FTAG);
675 677 return (err);
676 678 }
677 679
678 680 int
679 681 dmu_free_long_object(objset_t *os, uint64_t object)
680 682 {
681 683 dmu_tx_t *tx;
682 684 int err;
683 685
684 686 err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
685 687 if (err != 0)
686 688 return (err);
687 689
688 690 tx = dmu_tx_create(os);
689 691 dmu_tx_hold_bonus(tx, object);
690 692 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
691 693 err = dmu_tx_assign(tx, TXG_WAIT);
692 694 if (err == 0) {
693 695 err = dmu_object_free(os, object, tx);
694 696 dmu_tx_commit(tx);
695 697 } else {
696 698 dmu_tx_abort(tx);
697 699 }
698 700
699 701 return (err);
700 702 }
701 703
702 704 int
703 705 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
704 706 uint64_t size, dmu_tx_t *tx)
705 707 {
706 708 dnode_t *dn;
707 709 int err = dnode_hold(os, object, FTAG, &dn);
708 710 if (err)
709 711 return (err);
710 712 ASSERT(offset < UINT64_MAX);
711 713 ASSERT(size == -1ULL || size <= UINT64_MAX - offset);
712 714 dnode_free_range(dn, offset, size, tx);
713 715 dnode_rele(dn, FTAG);
714 716 return (0);
715 717 }
716 718
717 719 int
718 720 dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
719 721 void *buf, uint32_t flags)
720 722 {
721 723 dnode_t *dn;
722 724 dmu_buf_t **dbp;
723 725 int numbufs, err;
724 726
725 727 err = dnode_hold(os, object, FTAG, &dn);
726 728 if (err)
727 729 return (err);
728 730
729 731 /*
730 732 * Deal with odd block sizes, where there can't be data past the first
731 733 * block. If we ever do the tail block optimization, we will need to
732 734 * handle that here as well.
733 735 */
734 736 if (dn->dn_maxblkid == 0) {
735 737 int newsz = offset > dn->dn_datablksz ? 0 :
736 738 MIN(size, dn->dn_datablksz - offset);
737 739 bzero((char *)buf + newsz, size - newsz);
738 740 size = newsz;
739 741 }
740 742
741 743 while (size > 0) {
742 744 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
743 745 int i;
744 746
745 747 /*
746 748 * NB: we could do this block-at-a-time, but it's nice
747 749 * to be reading in parallel.
748 750 */
749 751 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
750 752 TRUE, FTAG, &numbufs, &dbp, flags);
751 753 if (err)
752 754 break;
753 755
754 756 for (i = 0; i < numbufs; i++) {
755 757 int tocpy;
756 758 int bufoff;
757 759 dmu_buf_t *db = dbp[i];
758 760
759 761 ASSERT(size > 0);
760 762
761 763 bufoff = offset - db->db_offset;
762 764 tocpy = (int)MIN(db->db_size - bufoff, size);
763 765
764 766 bcopy((char *)db->db_data + bufoff, buf, tocpy);
765 767
766 768 offset += tocpy;
767 769 size -= tocpy;
768 770 buf = (char *)buf + tocpy;
769 771 }
770 772 dmu_buf_rele_array(dbp, numbufs, FTAG);
771 773 }
772 774 dnode_rele(dn, FTAG);
773 775 return (err);
774 776 }
775 777
776 778 void
777 779 dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
778 780 const void *buf, dmu_tx_t *tx)
779 781 {
780 782 dmu_buf_t **dbp;
781 783 int numbufs, i;
782 784
783 785 if (size == 0)
784 786 return;
785 787
786 788 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
787 789 FALSE, FTAG, &numbufs, &dbp));
788 790
789 791 for (i = 0; i < numbufs; i++) {
790 792 int tocpy;
791 793 int bufoff;
792 794 dmu_buf_t *db = dbp[i];
793 795
794 796 ASSERT(size > 0);
795 797
796 798 bufoff = offset - db->db_offset;
797 799 tocpy = (int)MIN(db->db_size - bufoff, size);
798 800
799 801 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
800 802
801 803 if (tocpy == db->db_size)
802 804 dmu_buf_will_fill(db, tx);
803 805 else
804 806 dmu_buf_will_dirty(db, tx);
805 807
806 808 bcopy(buf, (char *)db->db_data + bufoff, tocpy);
807 809
808 810 if (tocpy == db->db_size)
809 811 dmu_buf_fill_done(db, tx);
810 812
811 813 offset += tocpy;
812 814 size -= tocpy;
813 815 buf = (char *)buf + tocpy;
814 816 }
815 817 dmu_buf_rele_array(dbp, numbufs, FTAG);
816 818 }
817 819
818 820 void
819 821 dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
820 822 dmu_tx_t *tx)
821 823 {
822 824 dmu_buf_t **dbp;
823 825 int numbufs, i;
824 826
825 827 if (size == 0)
826 828 return;
827 829
828 830 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
829 831 FALSE, FTAG, &numbufs, &dbp));
830 832
831 833 for (i = 0; i < numbufs; i++) {
832 834 dmu_buf_t *db = dbp[i];
833 835
834 836 dmu_buf_will_not_fill(db, tx);
835 837 }
836 838 dmu_buf_rele_array(dbp, numbufs, FTAG);
837 839 }
838 840
839 841 /*
840 842 * DMU support for xuio
841 843 */
842 844 kstat_t *xuio_ksp = NULL;
843 845
844 846 int
845 847 dmu_xuio_init(xuio_t *xuio, int nblk)
846 848 {
847 849 dmu_xuio_t *priv;
848 850 uio_t *uio = &xuio->xu_uio;
849 851
850 852 uio->uio_iovcnt = nblk;
851 853 uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
852 854
853 855 priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
854 856 priv->cnt = nblk;
855 857 priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
856 858 priv->iovp = uio->uio_iov;
857 859 XUIO_XUZC_PRIV(xuio) = priv;
858 860
859 861 if (XUIO_XUZC_RW(xuio) == UIO_READ)
860 862 XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk);
861 863 else
862 864 XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk);
863 865
864 866 return (0);
865 867 }
866 868
867 869 void
868 870 dmu_xuio_fini(xuio_t *xuio)
869 871 {
870 872 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
871 873 int nblk = priv->cnt;
872 874
873 875 kmem_free(priv->iovp, nblk * sizeof (iovec_t));
874 876 kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *));
875 877 kmem_free(priv, sizeof (dmu_xuio_t));
876 878
877 879 if (XUIO_XUZC_RW(xuio) == UIO_READ)
878 880 XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk);
879 881 else
880 882 XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk);
881 883 }
882 884
883 885 /*
884 886 * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf }
885 887 * and increase priv->next by 1.
886 888 */
887 889 int
888 890 dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n)
889 891 {
890 892 struct iovec *iov;
891 893 uio_t *uio = &xuio->xu_uio;
892 894 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
893 895 int i = priv->next++;
894 896
895 897 ASSERT(i < priv->cnt);
896 898 ASSERT(off + n <= arc_buf_size(abuf));
897 899 iov = uio->uio_iov + i;
898 900 iov->iov_base = (char *)abuf->b_data + off;
899 901 iov->iov_len = n;
900 902 priv->bufs[i] = abuf;
901 903 return (0);
902 904 }
903 905
904 906 int
905 907 dmu_xuio_cnt(xuio_t *xuio)
906 908 {
907 909 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
908 910 return (priv->cnt);
909 911 }
910 912
911 913 arc_buf_t *
912 914 dmu_xuio_arcbuf(xuio_t *xuio, int i)
913 915 {
914 916 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
915 917
916 918 ASSERT(i < priv->cnt);
917 919 return (priv->bufs[i]);
918 920 }
919 921
920 922 void
921 923 dmu_xuio_clear(xuio_t *xuio, int i)
922 924 {
923 925 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
924 926
925 927 ASSERT(i < priv->cnt);
926 928 priv->bufs[i] = NULL;
927 929 }
928 930
929 931 static void
930 932 xuio_stat_init(void)
931 933 {
932 934 xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc",
933 935 KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t),
934 936 KSTAT_FLAG_VIRTUAL);
935 937 if (xuio_ksp != NULL) {
936 938 xuio_ksp->ks_data = &xuio_stats;
937 939 kstat_install(xuio_ksp);
938 940 }
939 941 }
940 942
941 943 static void
942 944 xuio_stat_fini(void)
943 945 {
944 946 if (xuio_ksp != NULL) {
945 947 kstat_delete(xuio_ksp);
946 948 xuio_ksp = NULL;
947 949 }
948 950 }
949 951
950 952 void
951 953 xuio_stat_wbuf_copied()
952 954 {
953 955 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
954 956 }
955 957
956 958 void
957 959 xuio_stat_wbuf_nocopy()
958 960 {
959 961 XUIOSTAT_BUMP(xuiostat_wbuf_nocopy);
960 962 }
961 963
962 964 #ifdef _KERNEL
963 965 int
964 966 dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
965 967 {
966 968 dmu_buf_t **dbp;
967 969 int numbufs, i, err;
968 970 xuio_t *xuio = NULL;
969 971
970 972 /*
971 973 * NB: we could do this block-at-a-time, but it's nice
972 974 * to be reading in parallel.
973 975 */
974 976 err = dmu_buf_hold_array(os, object, uio->uio_loffset, size, TRUE, FTAG,
975 977 &numbufs, &dbp);
976 978 if (err)
977 979 return (err);
978 980
979 981 if (uio->uio_extflg == UIO_XUIO)
980 982 xuio = (xuio_t *)uio;
981 983
982 984 for (i = 0; i < numbufs; i++) {
983 985 int tocpy;
984 986 int bufoff;
985 987 dmu_buf_t *db = dbp[i];
986 988
987 989 ASSERT(size > 0);
988 990
989 991 bufoff = uio->uio_loffset - db->db_offset;
990 992 tocpy = (int)MIN(db->db_size - bufoff, size);
991 993
992 994 if (xuio) {
993 995 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
994 996 arc_buf_t *dbuf_abuf = dbi->db_buf;
995 997 arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
996 998 err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
997 999 if (!err) {
998 1000 uio->uio_resid -= tocpy;
999 1001 uio->uio_loffset += tocpy;
1000 1002 }
1001 1003
1002 1004 if (abuf == dbuf_abuf)
1003 1005 XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
1004 1006 else
1005 1007 XUIOSTAT_BUMP(xuiostat_rbuf_copied);
1006 1008 } else {
1007 1009 err = uiomove((char *)db->db_data + bufoff, tocpy,
1008 1010 UIO_READ, uio);
1009 1011 }
1010 1012 if (err)
1011 1013 break;
1012 1014
1013 1015 size -= tocpy;
1014 1016 }
1015 1017 dmu_buf_rele_array(dbp, numbufs, FTAG);
1016 1018
1017 1019 return (err);
1018 1020 }
1019 1021
1020 1022 static int
1021 1023 dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
1022 1024 {
1023 1025 dmu_buf_t **dbp;
1024 1026 int numbufs;
1025 1027 int err = 0;
1026 1028 int i;
1027 1029
1028 1030 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1029 1031 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1030 1032 if (err)
1031 1033 return (err);
1032 1034
1033 1035 for (i = 0; i < numbufs; i++) {
1034 1036 int tocpy;
1035 1037 int bufoff;
1036 1038 dmu_buf_t *db = dbp[i];
1037 1039
1038 1040 ASSERT(size > 0);
1039 1041
1040 1042 bufoff = uio->uio_loffset - db->db_offset;
1041 1043 tocpy = (int)MIN(db->db_size - bufoff, size);
1042 1044
1043 1045 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1044 1046
1045 1047 if (tocpy == db->db_size)
1046 1048 dmu_buf_will_fill(db, tx);
1047 1049 else
1048 1050 dmu_buf_will_dirty(db, tx);
1049 1051
1050 1052 /*
1051 1053 * XXX uiomove could block forever (eg. nfs-backed
1052 1054 * pages). There needs to be a uiolockdown() function
1053 1055 * to lock the pages in memory, so that uiomove won't
1054 1056 * block.
1055 1057 */
1056 1058 err = uiomove((char *)db->db_data + bufoff, tocpy,
1057 1059 UIO_WRITE, uio);
1058 1060
1059 1061 if (tocpy == db->db_size)
1060 1062 dmu_buf_fill_done(db, tx);
1061 1063
1062 1064 if (err)
1063 1065 break;
1064 1066
1065 1067 size -= tocpy;
1066 1068 }
1067 1069
1068 1070 dmu_buf_rele_array(dbp, numbufs, FTAG);
1069 1071 return (err);
1070 1072 }
1071 1073
1072 1074 int
1073 1075 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
1074 1076 dmu_tx_t *tx)
1075 1077 {
1076 1078 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1077 1079 dnode_t *dn;
1078 1080 int err;
1079 1081
1080 1082 if (size == 0)
1081 1083 return (0);
1082 1084
1083 1085 DB_DNODE_ENTER(db);
1084 1086 dn = DB_DNODE(db);
1085 1087 err = dmu_write_uio_dnode(dn, uio, size, tx);
1086 1088 DB_DNODE_EXIT(db);
1087 1089
1088 1090 return (err);
1089 1091 }
1090 1092
1091 1093 int
1092 1094 dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
1093 1095 dmu_tx_t *tx)
1094 1096 {
1095 1097 dnode_t *dn;
1096 1098 int err;
1097 1099
1098 1100 if (size == 0)
1099 1101 return (0);
1100 1102
1101 1103 err = dnode_hold(os, object, FTAG, &dn);
1102 1104 if (err)
1103 1105 return (err);
1104 1106
1105 1107 err = dmu_write_uio_dnode(dn, uio, size, tx);
1106 1108
1107 1109 dnode_rele(dn, FTAG);
1108 1110
1109 1111 return (err);
1110 1112 }
1111 1113
1112 1114 int
1113 1115 dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1114 1116 page_t *pp, dmu_tx_t *tx)
1115 1117 {
1116 1118 dmu_buf_t **dbp;
1117 1119 int numbufs, i;
1118 1120 int err;
1119 1121
1120 1122 if (size == 0)
1121 1123 return (0);
1122 1124
1123 1125 err = dmu_buf_hold_array(os, object, offset, size,
1124 1126 FALSE, FTAG, &numbufs, &dbp);
1125 1127 if (err)
1126 1128 return (err);
1127 1129
1128 1130 for (i = 0; i < numbufs; i++) {
1129 1131 int tocpy, copied, thiscpy;
1130 1132 int bufoff;
1131 1133 dmu_buf_t *db = dbp[i];
1132 1134 caddr_t va;
1133 1135
1134 1136 ASSERT(size > 0);
1135 1137 ASSERT3U(db->db_size, >=, PAGESIZE);
1136 1138
1137 1139 bufoff = offset - db->db_offset;
1138 1140 tocpy = (int)MIN(db->db_size - bufoff, size);
1139 1141
1140 1142 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1141 1143
1142 1144 if (tocpy == db->db_size)
1143 1145 dmu_buf_will_fill(db, tx);
1144 1146 else
1145 1147 dmu_buf_will_dirty(db, tx);
1146 1148
1147 1149 for (copied = 0; copied < tocpy; copied += PAGESIZE) {
1148 1150 ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff);
1149 1151 thiscpy = MIN(PAGESIZE, tocpy - copied);
1150 1152 va = zfs_map_page(pp, S_READ);
1151 1153 bcopy(va, (char *)db->db_data + bufoff, thiscpy);
1152 1154 zfs_unmap_page(pp, va);
1153 1155 pp = pp->p_next;
1154 1156 bufoff += PAGESIZE;
1155 1157 }
1156 1158
1157 1159 if (tocpy == db->db_size)
1158 1160 dmu_buf_fill_done(db, tx);
1159 1161
1160 1162 offset += tocpy;
1161 1163 size -= tocpy;
1162 1164 }
1163 1165 dmu_buf_rele_array(dbp, numbufs, FTAG);
1164 1166 return (err);
1165 1167 }
1166 1168 #endif
1167 1169
1168 1170 /*
1169 1171 * Allocate a loaned anonymous arc buffer.
1170 1172 */
1171 1173 arc_buf_t *
1172 1174 dmu_request_arcbuf(dmu_buf_t *handle, int size)
1173 1175 {
1174 1176 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
1175 1177 spa_t *spa;
1176 1178
1177 1179 DB_GET_SPA(&spa, db);
1178 1180 return (arc_loan_buf(spa, size));
1179 1181 }
1180 1182
1181 1183 /*
1182 1184 * Free a loaned arc buffer.
1183 1185 */
1184 1186 void
1185 1187 dmu_return_arcbuf(arc_buf_t *buf)
1186 1188 {
1187 1189 arc_return_buf(buf, FTAG);
1188 1190 VERIFY(arc_buf_remove_ref(buf, FTAG));
1189 1191 }
1190 1192
1191 1193 /*
1192 1194 * When possible directly assign passed loaned arc buffer to a dbuf.
1193 1195 * If this is not possible copy the contents of passed arc buf via
1194 1196 * dmu_write().
1195 1197 */
1196 1198 void
1197 1199 dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
1198 1200 dmu_tx_t *tx)
1199 1201 {
1200 1202 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1201 1203 dnode_t *dn;
1202 1204 dmu_buf_impl_t *db;
1203 1205 uint32_t blksz = (uint32_t)arc_buf_size(buf);
1204 1206 uint64_t blkid;
1205 1207
1206 1208 DB_DNODE_ENTER(dbuf);
1207 1209 dn = DB_DNODE(dbuf);
1208 1210 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1209 1211 blkid = dbuf_whichblock(dn, offset);
1210 1212 VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
1211 1213 rw_exit(&dn->dn_struct_rwlock);
1212 1214 DB_DNODE_EXIT(dbuf);
1213 1215
1214 1216 if (offset == db->db.db_offset && blksz == db->db.db_size) {
1215 1217 dbuf_assign_arcbuf(db, buf, tx);
1216 1218 dbuf_rele(db, FTAG);
1217 1219 } else {
1218 1220 objset_t *os;
1219 1221 uint64_t object;
1220 1222
1221 1223 DB_DNODE_ENTER(dbuf);
1222 1224 dn = DB_DNODE(dbuf);
1223 1225 os = dn->dn_objset;
1224 1226 object = dn->dn_object;
1225 1227 DB_DNODE_EXIT(dbuf);
1226 1228
1227 1229 dbuf_rele(db, FTAG);
1228 1230 dmu_write(os, object, offset, blksz, buf->b_data, tx);
1229 1231 dmu_return_arcbuf(buf);
1230 1232 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1231 1233 }
1232 1234 }
1233 1235
1234 1236 typedef struct {
1235 1237 dbuf_dirty_record_t *dsa_dr;
1236 1238 dmu_sync_cb_t *dsa_done;
1237 1239 zgd_t *dsa_zgd;
1238 1240 dmu_tx_t *dsa_tx;
1239 1241 } dmu_sync_arg_t;
1240 1242
1241 1243 /* ARGSUSED */
1242 1244 static void
1243 1245 dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1244 1246 {
1245 1247 dmu_sync_arg_t *dsa = varg;
1246 1248 dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1247 1249 blkptr_t *bp = zio->io_bp;
1248 1250
1249 1251 if (zio->io_error == 0) {
1250 1252 if (BP_IS_HOLE(bp)) {
1251 1253 /*
1252 1254 * A block of zeros may compress to a hole, but the
1253 1255 * block size still needs to be known for replay.
1254 1256 */
1255 1257 BP_SET_LSIZE(bp, db->db_size);
1256 1258 } else {
1257 1259 ASSERT(BP_GET_LEVEL(bp) == 0);
1258 1260 bp->blk_fill = 1;
1259 1261 }
1260 1262 }
1261 1263 }
1262 1264
1263 1265 static void
1264 1266 dmu_sync_late_arrival_ready(zio_t *zio)
1265 1267 {
1266 1268 dmu_sync_ready(zio, NULL, zio->io_private);
1267 1269 }
1268 1270
1269 1271 /* ARGSUSED */
1270 1272 static void
1271 1273 dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1272 1274 {
1273 1275 dmu_sync_arg_t *dsa = varg;
1274 1276 dbuf_dirty_record_t *dr = dsa->dsa_dr;
1275 1277 dmu_buf_impl_t *db = dr->dr_dbuf;
1276 1278
1277 1279 mutex_enter(&db->db_mtx);
1278 1280 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1279 1281 if (zio->io_error == 0) {
1280 1282 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
1281 1283 if (dr->dt.dl.dr_nopwrite) {
1282 1284 blkptr_t *bp = zio->io_bp;
1283 1285 blkptr_t *bp_orig = &zio->io_bp_orig;
1284 1286 uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
1285 1287
1286 1288 ASSERT(BP_EQUAL(bp, bp_orig));
1287 1289 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
1288 1290 ASSERT(zio_checksum_table[chksum].ci_dedup);
1289 1291 }
1290 1292 dr->dt.dl.dr_overridden_by = *zio->io_bp;
1291 1293 dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1292 1294 dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
1293 1295 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by))
1294 1296 BP_ZERO(&dr->dt.dl.dr_overridden_by);
1295 1297 } else {
1296 1298 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1297 1299 }
1298 1300 cv_broadcast(&db->db_changed);
1299 1301 mutex_exit(&db->db_mtx);
1300 1302
1301 1303 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1302 1304
1303 1305 kmem_free(dsa, sizeof (*dsa));
1304 1306 }
1305 1307
1306 1308 static void
1307 1309 dmu_sync_late_arrival_done(zio_t *zio)
1308 1310 {
1309 1311 blkptr_t *bp = zio->io_bp;
1310 1312 dmu_sync_arg_t *dsa = zio->io_private;
1311 1313 blkptr_t *bp_orig = &zio->io_bp_orig;
1312 1314
1313 1315 if (zio->io_error == 0 && !BP_IS_HOLE(bp)) {
1314 1316 /*
1315 1317 * If we didn't allocate a new block (i.e. ZIO_FLAG_NOPWRITE)
1316 1318 * then there is nothing to do here. Otherwise, free the
1317 1319 * newly allocated block in this txg.
1318 1320 */
1319 1321 if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
1320 1322 ASSERT(BP_EQUAL(bp, bp_orig));
1321 1323 } else {
1322 1324 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
1323 1325 ASSERT(zio->io_bp->blk_birth == zio->io_txg);
1324 1326 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
1325 1327 zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
1326 1328 }
1327 1329 }
1328 1330
1329 1331 dmu_tx_commit(dsa->dsa_tx);
1330 1332
1331 1333 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1332 1334
1333 1335 kmem_free(dsa, sizeof (*dsa));
1334 1336 }
1335 1337
1336 1338 static int
1337 1339 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
1338 1340 zio_prop_t *zp, zbookmark_t *zb)
1339 1341 {
1340 1342 dmu_sync_arg_t *dsa;
1341 1343 dmu_tx_t *tx;
1342 1344
1343 1345 tx = dmu_tx_create(os);
1344 1346 dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
1345 1347 if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1346 1348 dmu_tx_abort(tx);
1347 1349 /* Make zl_get_data do txg_waited_synced() */
1348 1350 return (SET_ERROR(EIO));
|
↓ open down ↓ |
779 lines elided |
↑ open up ↑ |
1349 1351 }
1350 1352
1351 1353 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1352 1354 dsa->dsa_dr = NULL;
1353 1355 dsa->dsa_done = done;
1354 1356 dsa->dsa_zgd = zgd;
1355 1357 dsa->dsa_tx = tx;
1356 1358
1357 1359 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
1358 1360 zgd->zgd_db->db_data, zgd->zgd_db->db_size, zp,
1359 - dmu_sync_late_arrival_ready, dmu_sync_late_arrival_done, dsa,
1361 + dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done, dsa,
1360 1362 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
1361 1363
1362 1364 return (0);
1363 1365 }
1364 1366
1365 1367 /*
1366 1368 * Intent log support: sync the block associated with db to disk.
1367 1369 * N.B. and XXX: the caller is responsible for making sure that the
1368 1370 * data isn't changing while dmu_sync() is writing it.
1369 1371 *
1370 1372 * Return values:
1371 1373 *
1372 1374 * EEXIST: this txg has already been synced, so there's nothing to do.
1373 1375 * The caller should not log the write.
1374 1376 *
1375 1377 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
1376 1378 * The caller should not log the write.
1377 1379 *
1378 1380 * EALREADY: this block is already in the process of being synced.
1379 1381 * The caller should track its progress (somehow).
1380 1382 *
1381 1383 * EIO: could not do the I/O.
1382 1384 * The caller should do a txg_wait_synced().
1383 1385 *
1384 1386 * 0: the I/O has been initiated.
1385 1387 * The caller should log this blkptr in the done callback.
1386 1388 * It is possible that the I/O will fail, in which case
1387 1389 * the error will be reported to the done callback and
1388 1390 * propagated to pio from zio_done().
1389 1391 */
1390 1392 int
1391 1393 dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
1392 1394 {
1393 1395 blkptr_t *bp = zgd->zgd_bp;
1394 1396 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
1395 1397 objset_t *os = db->db_objset;
1396 1398 dsl_dataset_t *ds = os->os_dsl_dataset;
1397 1399 dbuf_dirty_record_t *dr;
1398 1400 dmu_sync_arg_t *dsa;
1399 1401 zbookmark_t zb;
1400 1402 zio_prop_t zp;
1401 1403 dnode_t *dn;
1402 1404
1403 1405 ASSERT(pio != NULL);
1404 1406 ASSERT(txg != 0);
1405 1407
1406 1408 SET_BOOKMARK(&zb, ds->ds_object,
1407 1409 db->db.db_object, db->db_level, db->db_blkid);
1408 1410
1409 1411 DB_DNODE_ENTER(db);
1410 1412 dn = DB_DNODE(db);
1411 1413 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
1412 1414 DB_DNODE_EXIT(db);
1413 1415
1414 1416 /*
1415 1417 * If we're frozen (running ziltest), we always need to generate a bp.
1416 1418 */
1417 1419 if (txg > spa_freeze_txg(os->os_spa))
1418 1420 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1419 1421
1420 1422 /*
1421 1423 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
1422 1424 * and us. If we determine that this txg is not yet syncing,
1423 1425 * but it begins to sync a moment later, that's OK because the
1424 1426 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
1425 1427 */
1426 1428 mutex_enter(&db->db_mtx);
1427 1429
1428 1430 if (txg <= spa_last_synced_txg(os->os_spa)) {
1429 1431 /*
1430 1432 * This txg has already synced. There's nothing to do.
1431 1433 */
1432 1434 mutex_exit(&db->db_mtx);
1433 1435 return (SET_ERROR(EEXIST));
1434 1436 }
1435 1437
1436 1438 if (txg <= spa_syncing_txg(os->os_spa)) {
1437 1439 /*
1438 1440 * This txg is currently syncing, so we can't mess with
1439 1441 * the dirty record anymore; just write a new log block.
1440 1442 */
1441 1443 mutex_exit(&db->db_mtx);
1442 1444 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1443 1445 }
1444 1446
1445 1447 dr = db->db_last_dirty;
1446 1448 while (dr && dr->dr_txg != txg)
1447 1449 dr = dr->dr_next;
1448 1450
1449 1451 if (dr == NULL) {
1450 1452 /*
1451 1453 * There's no dr for this dbuf, so it must have been freed.
1452 1454 * There's no need to log writes to freed blocks, so we're done.
1453 1455 */
1454 1456 mutex_exit(&db->db_mtx);
1455 1457 return (SET_ERROR(ENOENT));
1456 1458 }
1457 1459
1458 1460 ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
1459 1461
1460 1462 /*
1461 1463 * Assume the on-disk data is X, the current syncing data is Y,
1462 1464 * and the current in-memory data is Z (currently in dmu_sync).
1463 1465 * X and Z are identical but Y is has been modified. Normally,
1464 1466 * when X and Z are the same we will perform a nopwrite but if Y
1465 1467 * is different we must disable nopwrite since the resulting write
1466 1468 * of Y to disk can free the block containing X. If we allowed a
1467 1469 * nopwrite to occur the block pointing to Z would reference a freed
1468 1470 * block. Since this is a rare case we simplify this by disabling
1469 1471 * nopwrite if the current dmu_sync-ing dbuf has been modified in
1470 1472 * a previous transaction.
1471 1473 */
1472 1474 if (dr->dr_next)
1473 1475 zp.zp_nopwrite = B_FALSE;
1474 1476
1475 1477 ASSERT(dr->dr_txg == txg);
1476 1478 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
1477 1479 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1478 1480 /*
1479 1481 * We have already issued a sync write for this buffer,
1480 1482 * or this buffer has already been synced. It could not
1481 1483 * have been dirtied since, or we would have cleared the state.
1482 1484 */
1483 1485 mutex_exit(&db->db_mtx);
1484 1486 return (SET_ERROR(EALREADY));
1485 1487 }
1486 1488
1487 1489 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
1488 1490 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
|
↓ open down ↓ |
119 lines elided |
↑ open up ↑ |
1489 1491 mutex_exit(&db->db_mtx);
1490 1492
1491 1493 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1492 1494 dsa->dsa_dr = dr;
1493 1495 dsa->dsa_done = done;
1494 1496 dsa->dsa_zgd = zgd;
1495 1497 dsa->dsa_tx = NULL;
1496 1498
1497 1499 zio_nowait(arc_write(pio, os->os_spa, txg,
1498 1500 bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
1499 - DBUF_IS_L2COMPRESSIBLE(db), &zp, dmu_sync_ready, dmu_sync_done,
1500 - dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
1501 + DBUF_IS_L2COMPRESSIBLE(db), &zp, dmu_sync_ready,
1502 + NULL, dmu_sync_done, dsa, ZIO_PRIORITY_SYNC_WRITE,
1503 + ZIO_FLAG_CANFAIL, &zb));
1501 1504
1502 1505 return (0);
1503 1506 }
1504 1507
1505 1508 int
1506 1509 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
1507 1510 dmu_tx_t *tx)
1508 1511 {
1509 1512 dnode_t *dn;
1510 1513 int err;
1511 1514
1512 1515 err = dnode_hold(os, object, FTAG, &dn);
1513 1516 if (err)
1514 1517 return (err);
1515 1518 err = dnode_set_blksz(dn, size, ibs, tx);
1516 1519 dnode_rele(dn, FTAG);
1517 1520 return (err);
1518 1521 }
1519 1522
1520 1523 void
1521 1524 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
1522 1525 dmu_tx_t *tx)
1523 1526 {
1524 1527 dnode_t *dn;
1525 1528
1526 1529 /* XXX assumes dnode_hold will not get an i/o error */
1527 1530 (void) dnode_hold(os, object, FTAG, &dn);
1528 1531 ASSERT(checksum < ZIO_CHECKSUM_FUNCTIONS);
1529 1532 dn->dn_checksum = checksum;
1530 1533 dnode_setdirty(dn, tx);
1531 1534 dnode_rele(dn, FTAG);
1532 1535 }
1533 1536
1534 1537 void
1535 1538 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
1536 1539 dmu_tx_t *tx)
1537 1540 {
1538 1541 dnode_t *dn;
1539 1542
1540 1543 /* XXX assumes dnode_hold will not get an i/o error */
1541 1544 (void) dnode_hold(os, object, FTAG, &dn);
1542 1545 ASSERT(compress < ZIO_COMPRESS_FUNCTIONS);
1543 1546 dn->dn_compress = compress;
1544 1547 dnode_setdirty(dn, tx);
1545 1548 dnode_rele(dn, FTAG);
1546 1549 }
1547 1550
1548 1551 int zfs_mdcomp_disable = 0;
1549 1552
1550 1553 void
1551 1554 dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
1552 1555 {
1553 1556 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
1554 1557 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
1555 1558 (wp & WP_SPILL));
1556 1559 enum zio_checksum checksum = os->os_checksum;
1557 1560 enum zio_compress compress = os->os_compress;
1558 1561 enum zio_checksum dedup_checksum = os->os_dedup_checksum;
1559 1562 boolean_t dedup = B_FALSE;
1560 1563 boolean_t nopwrite = B_FALSE;
1561 1564 boolean_t dedup_verify = os->os_dedup_verify;
1562 1565 int copies = os->os_copies;
1563 1566
1564 1567 /*
1565 1568 * We maintain different write policies for each of the following
1566 1569 * types of data:
1567 1570 * 1. metadata
1568 1571 * 2. preallocated blocks (i.e. level-0 blocks of a dump device)
1569 1572 * 3. all other level 0 blocks
1570 1573 */
1571 1574 if (ismd) {
1572 1575 /*
1573 1576 * XXX -- we should design a compression algorithm
1574 1577 * that specializes in arrays of bps.
1575 1578 */
1576 1579 compress = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY :
1577 1580 ZIO_COMPRESS_LZJB;
1578 1581
1579 1582 /*
1580 1583 * Metadata always gets checksummed. If the data
1581 1584 * checksum is multi-bit correctable, and it's not a
1582 1585 * ZBT-style checksum, then it's suitable for metadata
1583 1586 * as well. Otherwise, the metadata checksum defaults
1584 1587 * to fletcher4.
1585 1588 */
1586 1589 if (zio_checksum_table[checksum].ci_correctable < 1 ||
1587 1590 zio_checksum_table[checksum].ci_eck)
1588 1591 checksum = ZIO_CHECKSUM_FLETCHER_4;
1589 1592 } else if (wp & WP_NOFILL) {
1590 1593 ASSERT(level == 0);
1591 1594
1592 1595 /*
1593 1596 * If we're writing preallocated blocks, we aren't actually
1594 1597 * writing them so don't set any policy properties. These
1595 1598 * blocks are currently only used by an external subsystem
1596 1599 * outside of zfs (i.e. dump) and not written by the zio
1597 1600 * pipeline.
1598 1601 */
1599 1602 compress = ZIO_COMPRESS_OFF;
1600 1603 checksum = ZIO_CHECKSUM_OFF;
1601 1604 } else {
1602 1605 compress = zio_compress_select(dn->dn_compress, compress);
1603 1606
1604 1607 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
1605 1608 zio_checksum_select(dn->dn_checksum, checksum) :
1606 1609 dedup_checksum;
1607 1610
1608 1611 /*
1609 1612 * Determine dedup setting. If we are in dmu_sync(),
1610 1613 * we won't actually dedup now because that's all
1611 1614 * done in syncing context; but we do want to use the
1612 1615 * dedup checkum. If the checksum is not strong
1613 1616 * enough to ensure unique signatures, force
1614 1617 * dedup_verify.
1615 1618 */
1616 1619 if (dedup_checksum != ZIO_CHECKSUM_OFF) {
1617 1620 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
1618 1621 if (!zio_checksum_table[checksum].ci_dedup)
1619 1622 dedup_verify = B_TRUE;
1620 1623 }
1621 1624
1622 1625 /*
1623 1626 * Enable nopwrite if we have a cryptographically secure
1624 1627 * checksum that has no known collisions (i.e. SHA-256)
1625 1628 * and compression is enabled. We don't enable nopwrite if
1626 1629 * dedup is enabled as the two features are mutually exclusive.
1627 1630 */
1628 1631 nopwrite = (!dedup && zio_checksum_table[checksum].ci_dedup &&
1629 1632 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
1630 1633 }
1631 1634
1632 1635 zp->zp_checksum = checksum;
1633 1636 zp->zp_compress = compress;
1634 1637 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
1635 1638 zp->zp_level = level;
1636 1639 zp->zp_copies = MIN(copies + ismd, spa_max_replication(os->os_spa));
1637 1640 zp->zp_dedup = dedup;
1638 1641 zp->zp_dedup_verify = dedup && dedup_verify;
1639 1642 zp->zp_nopwrite = nopwrite;
1640 1643 }
1641 1644
1642 1645 int
1643 1646 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
1644 1647 {
1645 1648 dnode_t *dn;
1646 1649 int i, err;
1647 1650
1648 1651 err = dnode_hold(os, object, FTAG, &dn);
1649 1652 if (err)
1650 1653 return (err);
1651 1654 /*
1652 1655 * Sync any current changes before
1653 1656 * we go trundling through the block pointers.
1654 1657 */
1655 1658 for (i = 0; i < TXG_SIZE; i++) {
1656 1659 if (list_link_active(&dn->dn_dirty_link[i]))
1657 1660 break;
1658 1661 }
1659 1662 if (i != TXG_SIZE) {
1660 1663 dnode_rele(dn, FTAG);
1661 1664 txg_wait_synced(dmu_objset_pool(os), 0);
1662 1665 err = dnode_hold(os, object, FTAG, &dn);
1663 1666 if (err)
1664 1667 return (err);
1665 1668 }
1666 1669
1667 1670 err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
1668 1671 dnode_rele(dn, FTAG);
1669 1672
1670 1673 return (err);
1671 1674 }
1672 1675
1673 1676 void
1674 1677 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
1675 1678 {
1676 1679 dnode_phys_t *dnp;
1677 1680
1678 1681 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1679 1682 mutex_enter(&dn->dn_mtx);
1680 1683
1681 1684 dnp = dn->dn_phys;
1682 1685
1683 1686 doi->doi_data_block_size = dn->dn_datablksz;
1684 1687 doi->doi_metadata_block_size = dn->dn_indblkshift ?
1685 1688 1ULL << dn->dn_indblkshift : 0;
1686 1689 doi->doi_type = dn->dn_type;
1687 1690 doi->doi_bonus_type = dn->dn_bonustype;
1688 1691 doi->doi_bonus_size = dn->dn_bonuslen;
1689 1692 doi->doi_indirection = dn->dn_nlevels;
1690 1693 doi->doi_checksum = dn->dn_checksum;
1691 1694 doi->doi_compress = dn->dn_compress;
1692 1695 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
1693 1696 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
1694 1697 doi->doi_fill_count = 0;
1695 1698 for (int i = 0; i < dnp->dn_nblkptr; i++)
1696 1699 doi->doi_fill_count += dnp->dn_blkptr[i].blk_fill;
1697 1700
1698 1701 mutex_exit(&dn->dn_mtx);
1699 1702 rw_exit(&dn->dn_struct_rwlock);
1700 1703 }
1701 1704
1702 1705 /*
1703 1706 * Get information on a DMU object.
1704 1707 * If doi is NULL, just indicates whether the object exists.
1705 1708 */
1706 1709 int
1707 1710 dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
1708 1711 {
1709 1712 dnode_t *dn;
1710 1713 int err = dnode_hold(os, object, FTAG, &dn);
1711 1714
1712 1715 if (err)
1713 1716 return (err);
1714 1717
1715 1718 if (doi != NULL)
1716 1719 dmu_object_info_from_dnode(dn, doi);
1717 1720
1718 1721 dnode_rele(dn, FTAG);
1719 1722 return (0);
1720 1723 }
1721 1724
1722 1725 /*
1723 1726 * As above, but faster; can be used when you have a held dbuf in hand.
1724 1727 */
1725 1728 void
1726 1729 dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
1727 1730 {
1728 1731 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1729 1732
1730 1733 DB_DNODE_ENTER(db);
1731 1734 dmu_object_info_from_dnode(DB_DNODE(db), doi);
1732 1735 DB_DNODE_EXIT(db);
1733 1736 }
1734 1737
1735 1738 /*
1736 1739 * Faster still when you only care about the size.
1737 1740 * This is specifically optimized for zfs_getattr().
1738 1741 */
1739 1742 void
1740 1743 dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
1741 1744 u_longlong_t *nblk512)
1742 1745 {
1743 1746 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1744 1747 dnode_t *dn;
1745 1748
1746 1749 DB_DNODE_ENTER(db);
1747 1750 dn = DB_DNODE(db);
1748 1751
1749 1752 *blksize = dn->dn_datablksz;
1750 1753 /* add 1 for dnode space */
1751 1754 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
1752 1755 SPA_MINBLOCKSHIFT) + 1;
1753 1756 DB_DNODE_EXIT(db);
1754 1757 }
1755 1758
1756 1759 void
1757 1760 byteswap_uint64_array(void *vbuf, size_t size)
1758 1761 {
1759 1762 uint64_t *buf = vbuf;
1760 1763 size_t count = size >> 3;
1761 1764 int i;
1762 1765
1763 1766 ASSERT((size & 7) == 0);
1764 1767
1765 1768 for (i = 0; i < count; i++)
1766 1769 buf[i] = BSWAP_64(buf[i]);
1767 1770 }
1768 1771
1769 1772 void
1770 1773 byteswap_uint32_array(void *vbuf, size_t size)
1771 1774 {
1772 1775 uint32_t *buf = vbuf;
1773 1776 size_t count = size >> 2;
1774 1777 int i;
1775 1778
1776 1779 ASSERT((size & 3) == 0);
1777 1780
1778 1781 for (i = 0; i < count; i++)
1779 1782 buf[i] = BSWAP_32(buf[i]);
1780 1783 }
1781 1784
1782 1785 void
1783 1786 byteswap_uint16_array(void *vbuf, size_t size)
1784 1787 {
1785 1788 uint16_t *buf = vbuf;
1786 1789 size_t count = size >> 1;
1787 1790 int i;
1788 1791
1789 1792 ASSERT((size & 1) == 0);
1790 1793
1791 1794 for (i = 0; i < count; i++)
1792 1795 buf[i] = BSWAP_16(buf[i]);
1793 1796 }
1794 1797
1795 1798 /* ARGSUSED */
1796 1799 void
1797 1800 byteswap_uint8_array(void *vbuf, size_t size)
1798 1801 {
1799 1802 }
1800 1803
1801 1804 void
1802 1805 dmu_init(void)
1803 1806 {
1804 1807 zfs_dbgmsg_init();
1805 1808 sa_cache_init();
1806 1809 xuio_stat_init();
1807 1810 dmu_objset_init();
1808 1811 dnode_init();
1809 1812 dbuf_init();
1810 1813 zfetch_init();
1811 1814 l2arc_init();
1812 1815 arc_init();
1813 1816 }
1814 1817
1815 1818 void
1816 1819 dmu_fini(void)
1817 1820 {
1818 1821 arc_fini(); /* arc depends on l2arc, so arc must go first */
1819 1822 l2arc_fini();
1820 1823 zfetch_fini();
1821 1824 dbuf_fini();
1822 1825 dnode_fini();
1823 1826 dmu_objset_fini();
1824 1827 xuio_stat_fini();
1825 1828 sa_cache_fini();
1826 1829 zfs_dbgmsg_fini();
1827 1830 }
|
↓ open down ↓ |
317 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX