Print this page
4047 panic from dbuf_free_range() from dmu_free_object() while doing zfs receive
Reviewed by: Adam Leventhal <ahl@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/dmu.c
+++ new/usr/src/uts/common/fs/zfs/dmu.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright (c) 2013 by Delphix. All rights reserved.
24 24 */
25 25
26 26 /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
27 27
28 28 #include <sys/dmu.h>
29 29 #include <sys/dmu_impl.h>
30 30 #include <sys/dmu_tx.h>
31 31 #include <sys/dbuf.h>
32 32 #include <sys/dnode.h>
33 33 #include <sys/zfs_context.h>
34 34 #include <sys/dmu_objset.h>
35 35 #include <sys/dmu_traverse.h>
36 36 #include <sys/dsl_dataset.h>
37 37 #include <sys/dsl_dir.h>
38 38 #include <sys/dsl_pool.h>
39 39 #include <sys/dsl_synctask.h>
40 40 #include <sys/dsl_prop.h>
41 41 #include <sys/dmu_zfetch.h>
42 42 #include <sys/zfs_ioctl.h>
43 43 #include <sys/zap.h>
44 44 #include <sys/zio_checksum.h>
45 45 #include <sys/zio_compress.h>
46 46 #include <sys/sa.h>
47 47 #ifdef _KERNEL
48 48 #include <sys/vmsystm.h>
49 49 #include <sys/zfs_znode.h>
50 50 #endif
51 51
52 52 /*
53 53 * Enable/disable nopwrite feature.
54 54 */
55 55 int zfs_nopwrite_enabled = 1;
56 56
57 57 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
58 58 { DMU_BSWAP_UINT8, TRUE, "unallocated" },
59 59 { DMU_BSWAP_ZAP, TRUE, "object directory" },
60 60 { DMU_BSWAP_UINT64, TRUE, "object array" },
61 61 { DMU_BSWAP_UINT8, TRUE, "packed nvlist" },
62 62 { DMU_BSWAP_UINT64, TRUE, "packed nvlist size" },
63 63 { DMU_BSWAP_UINT64, TRUE, "bpobj" },
64 64 { DMU_BSWAP_UINT64, TRUE, "bpobj header" },
65 65 { DMU_BSWAP_UINT64, TRUE, "SPA space map header" },
66 66 { DMU_BSWAP_UINT64, TRUE, "SPA space map" },
67 67 { DMU_BSWAP_UINT64, TRUE, "ZIL intent log" },
68 68 { DMU_BSWAP_DNODE, TRUE, "DMU dnode" },
69 69 { DMU_BSWAP_OBJSET, TRUE, "DMU objset" },
70 70 { DMU_BSWAP_UINT64, TRUE, "DSL directory" },
71 71 { DMU_BSWAP_ZAP, TRUE, "DSL directory child map"},
72 72 { DMU_BSWAP_ZAP, TRUE, "DSL dataset snap map" },
73 73 { DMU_BSWAP_ZAP, TRUE, "DSL props" },
74 74 { DMU_BSWAP_UINT64, TRUE, "DSL dataset" },
75 75 { DMU_BSWAP_ZNODE, TRUE, "ZFS znode" },
76 76 { DMU_BSWAP_OLDACL, TRUE, "ZFS V0 ACL" },
77 77 { DMU_BSWAP_UINT8, FALSE, "ZFS plain file" },
78 78 { DMU_BSWAP_ZAP, TRUE, "ZFS directory" },
79 79 { DMU_BSWAP_ZAP, TRUE, "ZFS master node" },
80 80 { DMU_BSWAP_ZAP, TRUE, "ZFS delete queue" },
81 81 { DMU_BSWAP_UINT8, FALSE, "zvol object" },
82 82 { DMU_BSWAP_ZAP, TRUE, "zvol prop" },
83 83 { DMU_BSWAP_UINT8, FALSE, "other uint8[]" },
84 84 { DMU_BSWAP_UINT64, FALSE, "other uint64[]" },
85 85 { DMU_BSWAP_ZAP, TRUE, "other ZAP" },
86 86 { DMU_BSWAP_ZAP, TRUE, "persistent error log" },
87 87 { DMU_BSWAP_UINT8, TRUE, "SPA history" },
88 88 { DMU_BSWAP_UINT64, TRUE, "SPA history offsets" },
89 89 { DMU_BSWAP_ZAP, TRUE, "Pool properties" },
90 90 { DMU_BSWAP_ZAP, TRUE, "DSL permissions" },
91 91 { DMU_BSWAP_ACL, TRUE, "ZFS ACL" },
92 92 { DMU_BSWAP_UINT8, TRUE, "ZFS SYSACL" },
93 93 { DMU_BSWAP_UINT8, TRUE, "FUID table" },
94 94 { DMU_BSWAP_UINT64, TRUE, "FUID table size" },
95 95 { DMU_BSWAP_ZAP, TRUE, "DSL dataset next clones"},
96 96 { DMU_BSWAP_ZAP, TRUE, "scan work queue" },
97 97 { DMU_BSWAP_ZAP, TRUE, "ZFS user/group used" },
98 98 { DMU_BSWAP_ZAP, TRUE, "ZFS user/group quota" },
99 99 { DMU_BSWAP_ZAP, TRUE, "snapshot refcount tags"},
100 100 { DMU_BSWAP_ZAP, TRUE, "DDT ZAP algorithm" },
101 101 { DMU_BSWAP_ZAP, TRUE, "DDT statistics" },
102 102 { DMU_BSWAP_UINT8, TRUE, "System attributes" },
103 103 { DMU_BSWAP_ZAP, TRUE, "SA master node" },
104 104 { DMU_BSWAP_ZAP, TRUE, "SA attr registration" },
105 105 { DMU_BSWAP_ZAP, TRUE, "SA attr layouts" },
106 106 { DMU_BSWAP_ZAP, TRUE, "scan translations" },
107 107 { DMU_BSWAP_UINT8, FALSE, "deduplicated block" },
108 108 { DMU_BSWAP_ZAP, TRUE, "DSL deadlist map" },
109 109 { DMU_BSWAP_UINT64, TRUE, "DSL deadlist map hdr" },
110 110 { DMU_BSWAP_ZAP, TRUE, "DSL dir clones" },
111 111 { DMU_BSWAP_UINT64, TRUE, "bpobj subobj" }
112 112 };
113 113
114 114 const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
115 115 { byteswap_uint8_array, "uint8" },
116 116 { byteswap_uint16_array, "uint16" },
117 117 { byteswap_uint32_array, "uint32" },
118 118 { byteswap_uint64_array, "uint64" },
119 119 { zap_byteswap, "zap" },
120 120 { dnode_buf_byteswap, "dnode" },
121 121 { dmu_objset_byteswap, "objset" },
122 122 { zfs_znode_byteswap, "znode" },
123 123 { zfs_oldacl_byteswap, "oldacl" },
124 124 { zfs_acl_byteswap, "acl" }
125 125 };
126 126
127 127 int
128 128 dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
129 129 void *tag, dmu_buf_t **dbp, int flags)
130 130 {
131 131 dnode_t *dn;
132 132 uint64_t blkid;
133 133 dmu_buf_impl_t *db;
134 134 int err;
135 135 int db_flags = DB_RF_CANFAIL;
136 136
137 137 if (flags & DMU_READ_NO_PREFETCH)
138 138 db_flags |= DB_RF_NOPREFETCH;
139 139
140 140 err = dnode_hold(os, object, FTAG, &dn);
141 141 if (err)
142 142 return (err);
143 143 blkid = dbuf_whichblock(dn, offset);
144 144 rw_enter(&dn->dn_struct_rwlock, RW_READER);
145 145 db = dbuf_hold(dn, blkid, tag);
146 146 rw_exit(&dn->dn_struct_rwlock);
147 147 if (db == NULL) {
148 148 err = SET_ERROR(EIO);
149 149 } else {
150 150 err = dbuf_read(db, NULL, db_flags);
151 151 if (err) {
152 152 dbuf_rele(db, tag);
153 153 db = NULL;
154 154 }
155 155 }
156 156
157 157 dnode_rele(dn, FTAG);
158 158 *dbp = &db->db; /* NULL db plus first field offset is NULL */
159 159 return (err);
160 160 }
161 161
162 162 int
163 163 dmu_bonus_max(void)
164 164 {
165 165 return (DN_MAX_BONUSLEN);
166 166 }
167 167
168 168 int
169 169 dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
170 170 {
171 171 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
172 172 dnode_t *dn;
173 173 int error;
174 174
175 175 DB_DNODE_ENTER(db);
176 176 dn = DB_DNODE(db);
177 177
178 178 if (dn->dn_bonus != db) {
179 179 error = SET_ERROR(EINVAL);
180 180 } else if (newsize < 0 || newsize > db_fake->db_size) {
181 181 error = SET_ERROR(EINVAL);
182 182 } else {
183 183 dnode_setbonuslen(dn, newsize, tx);
184 184 error = 0;
185 185 }
186 186
187 187 DB_DNODE_EXIT(db);
188 188 return (error);
189 189 }
190 190
191 191 int
192 192 dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
193 193 {
194 194 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
195 195 dnode_t *dn;
196 196 int error;
197 197
198 198 DB_DNODE_ENTER(db);
199 199 dn = DB_DNODE(db);
200 200
201 201 if (!DMU_OT_IS_VALID(type)) {
202 202 error = SET_ERROR(EINVAL);
203 203 } else if (dn->dn_bonus != db) {
204 204 error = SET_ERROR(EINVAL);
205 205 } else {
206 206 dnode_setbonus_type(dn, type, tx);
207 207 error = 0;
208 208 }
209 209
210 210 DB_DNODE_EXIT(db);
211 211 return (error);
212 212 }
213 213
214 214 dmu_object_type_t
215 215 dmu_get_bonustype(dmu_buf_t *db_fake)
216 216 {
217 217 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
218 218 dnode_t *dn;
219 219 dmu_object_type_t type;
220 220
221 221 DB_DNODE_ENTER(db);
222 222 dn = DB_DNODE(db);
223 223 type = dn->dn_bonustype;
224 224 DB_DNODE_EXIT(db);
225 225
226 226 return (type);
227 227 }
228 228
229 229 int
230 230 dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
231 231 {
232 232 dnode_t *dn;
233 233 int error;
234 234
235 235 error = dnode_hold(os, object, FTAG, &dn);
236 236 dbuf_rm_spill(dn, tx);
237 237 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
238 238 dnode_rm_spill(dn, tx);
239 239 rw_exit(&dn->dn_struct_rwlock);
240 240 dnode_rele(dn, FTAG);
241 241 return (error);
242 242 }
243 243
244 244 /*
245 245 * returns ENOENT, EIO, or 0.
246 246 */
247 247 int
248 248 dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
249 249 {
250 250 dnode_t *dn;
251 251 dmu_buf_impl_t *db;
252 252 int error;
253 253
254 254 error = dnode_hold(os, object, FTAG, &dn);
255 255 if (error)
256 256 return (error);
257 257
258 258 rw_enter(&dn->dn_struct_rwlock, RW_READER);
259 259 if (dn->dn_bonus == NULL) {
260 260 rw_exit(&dn->dn_struct_rwlock);
261 261 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
262 262 if (dn->dn_bonus == NULL)
263 263 dbuf_create_bonus(dn);
264 264 }
265 265 db = dn->dn_bonus;
266 266
267 267 /* as long as the bonus buf is held, the dnode will be held */
268 268 if (refcount_add(&db->db_holds, tag) == 1) {
269 269 VERIFY(dnode_add_ref(dn, db));
270 270 (void) atomic_inc_32_nv(&dn->dn_dbufs_count);
271 271 }
272 272
273 273 /*
274 274 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
275 275 * hold and incrementing the dbuf count to ensure that dnode_move() sees
276 276 * a dnode hold for every dbuf.
277 277 */
278 278 rw_exit(&dn->dn_struct_rwlock);
279 279
280 280 dnode_rele(dn, FTAG);
281 281
282 282 VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH));
283 283
284 284 *dbp = &db->db;
285 285 return (0);
286 286 }
287 287
288 288 /*
289 289 * returns ENOENT, EIO, or 0.
290 290 *
291 291 * This interface will allocate a blank spill dbuf when a spill blk
292 292 * doesn't already exist on the dnode.
293 293 *
294 294 * if you only want to find an already existing spill db, then
295 295 * dmu_spill_hold_existing() should be used.
296 296 */
297 297 int
298 298 dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
299 299 {
300 300 dmu_buf_impl_t *db = NULL;
301 301 int err;
302 302
303 303 if ((flags & DB_RF_HAVESTRUCT) == 0)
304 304 rw_enter(&dn->dn_struct_rwlock, RW_READER);
305 305
306 306 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
307 307
308 308 if ((flags & DB_RF_HAVESTRUCT) == 0)
309 309 rw_exit(&dn->dn_struct_rwlock);
310 310
311 311 ASSERT(db != NULL);
312 312 err = dbuf_read(db, NULL, flags);
313 313 if (err == 0)
314 314 *dbp = &db->db;
315 315 else
316 316 dbuf_rele(db, tag);
317 317 return (err);
318 318 }
319 319
320 320 int
321 321 dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
322 322 {
323 323 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
324 324 dnode_t *dn;
325 325 int err;
326 326
327 327 DB_DNODE_ENTER(db);
328 328 dn = DB_DNODE(db);
329 329
330 330 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
331 331 err = SET_ERROR(EINVAL);
332 332 } else {
333 333 rw_enter(&dn->dn_struct_rwlock, RW_READER);
334 334
335 335 if (!dn->dn_have_spill) {
336 336 err = SET_ERROR(ENOENT);
337 337 } else {
338 338 err = dmu_spill_hold_by_dnode(dn,
339 339 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
340 340 }
341 341
342 342 rw_exit(&dn->dn_struct_rwlock);
343 343 }
344 344
345 345 DB_DNODE_EXIT(db);
346 346 return (err);
347 347 }
348 348
349 349 int
350 350 dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
351 351 {
352 352 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
353 353 dnode_t *dn;
354 354 int err;
355 355
356 356 DB_DNODE_ENTER(db);
357 357 dn = DB_DNODE(db);
358 358 err = dmu_spill_hold_by_dnode(dn, DB_RF_CANFAIL, tag, dbp);
359 359 DB_DNODE_EXIT(db);
360 360
361 361 return (err);
362 362 }
363 363
364 364 /*
365 365 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
366 366 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
367 367 * and can induce severe lock contention when writing to several files
368 368 * whose dnodes are in the same block.
369 369 */
370 370 static int
371 371 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
372 372 int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
373 373 {
374 374 dsl_pool_t *dp = NULL;
375 375 dmu_buf_t **dbp;
376 376 uint64_t blkid, nblks, i;
377 377 uint32_t dbuf_flags;
378 378 int err;
379 379 zio_t *zio;
380 380 hrtime_t start;
381 381
382 382 ASSERT(length <= DMU_MAX_ACCESS);
383 383
384 384 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT;
385 385 if (flags & DMU_READ_NO_PREFETCH || length > zfetch_array_rd_sz)
386 386 dbuf_flags |= DB_RF_NOPREFETCH;
387 387
388 388 rw_enter(&dn->dn_struct_rwlock, RW_READER);
389 389 if (dn->dn_datablkshift) {
390 390 int blkshift = dn->dn_datablkshift;
391 391 nblks = (P2ROUNDUP(offset+length, 1ULL<<blkshift) -
392 392 P2ALIGN(offset, 1ULL<<blkshift)) >> blkshift;
393 393 } else {
394 394 if (offset + length > dn->dn_datablksz) {
395 395 zfs_panic_recover("zfs: accessing past end of object "
396 396 "%llx/%llx (size=%u access=%llu+%llu)",
397 397 (longlong_t)dn->dn_objset->
398 398 os_dsl_dataset->ds_object,
399 399 (longlong_t)dn->dn_object, dn->dn_datablksz,
400 400 (longlong_t)offset, (longlong_t)length);
401 401 rw_exit(&dn->dn_struct_rwlock);
402 402 return (SET_ERROR(EIO));
403 403 }
404 404 nblks = 1;
405 405 }
406 406 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
407 407
408 408 if (dn->dn_objset->os_dsl_dataset)
409 409 dp = dn->dn_objset->os_dsl_dataset->ds_dir->dd_pool;
410 410 start = gethrtime();
411 411 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
412 412 blkid = dbuf_whichblock(dn, offset);
413 413 for (i = 0; i < nblks; i++) {
414 414 dmu_buf_impl_t *db = dbuf_hold(dn, blkid+i, tag);
415 415 if (db == NULL) {
416 416 rw_exit(&dn->dn_struct_rwlock);
417 417 dmu_buf_rele_array(dbp, nblks, tag);
418 418 zio_nowait(zio);
419 419 return (SET_ERROR(EIO));
420 420 }
421 421 /* initiate async i/o */
422 422 if (read) {
423 423 (void) dbuf_read(db, zio, dbuf_flags);
424 424 }
425 425 dbp[i] = &db->db;
426 426 }
427 427 rw_exit(&dn->dn_struct_rwlock);
428 428
429 429 /* wait for async i/o */
430 430 err = zio_wait(zio);
431 431 /* track read overhead when we are in sync context */
432 432 if (dp && dsl_pool_sync_context(dp))
433 433 dp->dp_read_overhead += gethrtime() - start;
434 434 if (err) {
435 435 dmu_buf_rele_array(dbp, nblks, tag);
436 436 return (err);
437 437 }
438 438
439 439 /* wait for other io to complete */
440 440 if (read) {
441 441 for (i = 0; i < nblks; i++) {
442 442 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
443 443 mutex_enter(&db->db_mtx);
444 444 while (db->db_state == DB_READ ||
445 445 db->db_state == DB_FILL)
446 446 cv_wait(&db->db_changed, &db->db_mtx);
447 447 if (db->db_state == DB_UNCACHED)
448 448 err = SET_ERROR(EIO);
449 449 mutex_exit(&db->db_mtx);
450 450 if (err) {
451 451 dmu_buf_rele_array(dbp, nblks, tag);
452 452 return (err);
453 453 }
454 454 }
455 455 }
456 456
457 457 *numbufsp = nblks;
458 458 *dbpp = dbp;
459 459 return (0);
460 460 }
461 461
462 462 static int
463 463 dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
464 464 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
465 465 {
466 466 dnode_t *dn;
467 467 int err;
468 468
469 469 err = dnode_hold(os, object, FTAG, &dn);
470 470 if (err)
471 471 return (err);
472 472
473 473 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
474 474 numbufsp, dbpp, DMU_READ_PREFETCH);
475 475
476 476 dnode_rele(dn, FTAG);
477 477
478 478 return (err);
479 479 }
480 480
481 481 int
482 482 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
483 483 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
484 484 {
485 485 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
486 486 dnode_t *dn;
487 487 int err;
488 488
489 489 DB_DNODE_ENTER(db);
490 490 dn = DB_DNODE(db);
491 491 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
492 492 numbufsp, dbpp, DMU_READ_PREFETCH);
493 493 DB_DNODE_EXIT(db);
494 494
495 495 return (err);
496 496 }
497 497
498 498 void
499 499 dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
500 500 {
501 501 int i;
502 502 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
503 503
504 504 if (numbufs == 0)
505 505 return;
506 506
507 507 for (i = 0; i < numbufs; i++) {
508 508 if (dbp[i])
509 509 dbuf_rele(dbp[i], tag);
510 510 }
511 511
512 512 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
513 513 }
514 514
515 515 void
516 516 dmu_prefetch(objset_t *os, uint64_t object, uint64_t offset, uint64_t len)
517 517 {
518 518 dnode_t *dn;
519 519 uint64_t blkid;
520 520 int nblks, i, err;
521 521
522 522 if (zfs_prefetch_disable)
523 523 return;
524 524
525 525 if (len == 0) { /* they're interested in the bonus buffer */
526 526 dn = DMU_META_DNODE(os);
527 527
528 528 if (object == 0 || object >= DN_MAX_OBJECT)
529 529 return;
530 530
531 531 rw_enter(&dn->dn_struct_rwlock, RW_READER);
532 532 blkid = dbuf_whichblock(dn, object * sizeof (dnode_phys_t));
533 533 dbuf_prefetch(dn, blkid);
534 534 rw_exit(&dn->dn_struct_rwlock);
535 535 return;
536 536 }
537 537
538 538 /*
539 539 * XXX - Note, if the dnode for the requested object is not
540 540 * already cached, we will do a *synchronous* read in the
541 541 * dnode_hold() call. The same is true for any indirects.
542 542 */
543 543 err = dnode_hold(os, object, FTAG, &dn);
544 544 if (err != 0)
545 545 return;
546 546
547 547 rw_enter(&dn->dn_struct_rwlock, RW_READER);
548 548 if (dn->dn_datablkshift) {
549 549 int blkshift = dn->dn_datablkshift;
550 550 nblks = (P2ROUNDUP(offset+len, 1<<blkshift) -
551 551 P2ALIGN(offset, 1<<blkshift)) >> blkshift;
552 552 } else {
553 553 nblks = (offset < dn->dn_datablksz);
554 554 }
555 555
556 556 if (nblks != 0) {
557 557 blkid = dbuf_whichblock(dn, offset);
558 558 for (i = 0; i < nblks; i++)
559 559 dbuf_prefetch(dn, blkid+i);
560 560 }
561 561
|
↓ open down ↓ |
561 lines elided |
↑ open up ↑ |
562 562 rw_exit(&dn->dn_struct_rwlock);
563 563
564 564 dnode_rele(dn, FTAG);
565 565 }
566 566
567 567 /*
568 568 * Get the next "chunk" of file data to free. We traverse the file from
569 569 * the end so that the file gets shorter over time (if we crashes in the
570 570 * middle, this will leave us in a better state). We find allocated file
571 571 * data by simply searching the allocated level 1 indirects.
572 + *
573 + * On input, *start should be the first offset that does not need to be
574 + * freed (e.g. "offset + length"). On return, *start will be the first
575 + * offset that should be freed.
572 576 */
573 577 static int
574 -get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t limit)
578 +get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum)
575 579 {
576 - uint64_t len = *start - limit;
577 - uint64_t blkcnt = 0;
578 - uint64_t maxblks = DMU_MAX_ACCESS / (1ULL << (dn->dn_indblkshift + 1));
580 + uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
581 + /* bytes of data covered by a level-1 indirect block */
579 582 uint64_t iblkrange =
580 583 dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
581 584
582 - ASSERT(limit <= *start);
585 + ASSERT3U(minimum, <=, *start);
583 586
584 - if (len <= iblkrange * maxblks) {
585 - *start = limit;
587 + if (*start - minimum <= iblkrange * maxblks) {
588 + *start = minimum;
586 589 return (0);
587 590 }
588 591 ASSERT(ISP2(iblkrange));
589 592
590 - while (*start > limit && blkcnt < maxblks) {
593 + for (uint64_t blks = 0; *start > minimum && blks < maxblks; blks++) {
591 594 int err;
592 595
593 - /* find next allocated L1 indirect */
596 + /*
597 + * dnode_next_offset(BACKWARDS) will find an allocated L1
598 + * indirect block at or before the input offset. We must
599 + * decrement *start so that it is at the end of the region
600 + * to search.
601 + */
602 + (*start)--;
594 603 err = dnode_next_offset(dn,
595 604 DNODE_FIND_BACKWARDS, start, 2, 1, 0);
596 605
597 - /* if there are no more, then we are done */
606 + /* if there are no indirect blocks before start, we are done */
598 607 if (err == ESRCH) {
599 - *start = limit;
600 - return (0);
601 - } else if (err) {
608 + *start = minimum;
609 + break;
610 + } else if (err != 0) {
602 611 return (err);
603 612 }
604 - blkcnt += 1;
605 613
606 - /* reset offset to end of "next" block back */
614 + /* set start to the beginning of this L1 indirect */
607 615 *start = P2ALIGN(*start, iblkrange);
608 - if (*start <= limit)
609 - *start = limit;
610 - else
611 - *start -= 1;
612 616 }
617 + if (*start < minimum)
618 + *start = minimum;
613 619 return (0);
614 620 }
615 621
616 622 static int
617 623 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
618 - uint64_t length, boolean_t free_dnode)
624 + uint64_t length)
619 625 {
620 - dmu_tx_t *tx;
621 - uint64_t object_size, start, end, len;
622 - boolean_t trunc = (length == DMU_OBJECT_END);
623 - int align, err;
626 + uint64_t object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
627 + int err;
624 628
625 - align = 1 << dn->dn_datablkshift;
626 - ASSERT(align > 0);
627 - object_size = align == 1 ? dn->dn_datablksz :
628 - (dn->dn_maxblkid + 1) << dn->dn_datablkshift;
629 -
630 - end = offset + length;
631 - if (trunc || end > object_size)
632 - end = object_size;
633 - if (end <= offset)
629 + if (offset >= object_size)
634 630 return (0);
635 - length = end - offset;
636 631
637 - while (length) {
638 - start = end;
639 - /* assert(offset <= start) */
640 - err = get_next_chunk(dn, &start, offset);
632 + if (length == DMU_OBJECT_END || offset + length > object_size)
633 + length = object_size - offset;
634 +
635 + while (length != 0) {
636 + uint64_t chunk_end, chunk_begin;
637 +
638 + chunk_end = chunk_begin = offset + length;
639 +
640 + /* move chunk_begin backwards to the beginning of this chunk */
641 + err = get_next_chunk(dn, &chunk_begin, offset);
641 642 if (err)
642 643 return (err);
643 - len = trunc ? DMU_OBJECT_END : end - start;
644 + ASSERT3U(chunk_begin, >=, offset);
645 + ASSERT3U(chunk_begin, <=, chunk_end);
644 646
645 - tx = dmu_tx_create(os);
646 - dmu_tx_hold_free(tx, dn->dn_object, start, len);
647 + dmu_tx_t *tx = dmu_tx_create(os);
648 + dmu_tx_hold_free(tx, dn->dn_object,
649 + chunk_begin, chunk_end - chunk_begin);
647 650 err = dmu_tx_assign(tx, TXG_WAIT);
648 651 if (err) {
649 652 dmu_tx_abort(tx);
650 653 return (err);
651 654 }
652 -
653 - dnode_free_range(dn, start, trunc ? -1 : len, tx);
654 -
655 - if (start == 0 && free_dnode) {
656 - ASSERT(trunc);
657 - dnode_free(dn, tx);
658 - }
659 -
660 - length -= end - start;
661 -
655 + dnode_free_range(dn, chunk_begin, chunk_end - chunk_begin, tx);
662 656 dmu_tx_commit(tx);
663 - end = start;
657 +
658 + length -= chunk_end - chunk_begin;
664 659 }
665 660 return (0);
666 661 }
667 662
668 663 int
669 664 dmu_free_long_range(objset_t *os, uint64_t object,
670 665 uint64_t offset, uint64_t length)
671 666 {
672 667 dnode_t *dn;
673 668 int err;
674 669
675 670 err = dnode_hold(os, object, FTAG, &dn);
676 671 if (err != 0)
677 672 return (err);
678 - err = dmu_free_long_range_impl(os, dn, offset, length, FALSE);
673 + err = dmu_free_long_range_impl(os, dn, offset, length);
679 674 dnode_rele(dn, FTAG);
680 675 return (err);
681 676 }
682 677
683 678 int
684 -dmu_free_object(objset_t *os, uint64_t object)
679 +dmu_free_long_object(objset_t *os, uint64_t object)
685 680 {
686 - dnode_t *dn;
687 681 dmu_tx_t *tx;
688 682 int err;
689 683
690 - err = dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED,
691 - FTAG, &dn);
684 + err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
692 685 if (err != 0)
693 686 return (err);
694 - if (dn->dn_nlevels == 1) {
695 - tx = dmu_tx_create(os);
696 - dmu_tx_hold_bonus(tx, object);
697 - dmu_tx_hold_free(tx, dn->dn_object, 0, DMU_OBJECT_END);
698 - err = dmu_tx_assign(tx, TXG_WAIT);
699 - if (err == 0) {
700 - dnode_free_range(dn, 0, DMU_OBJECT_END, tx);
701 - dnode_free(dn, tx);
702 - dmu_tx_commit(tx);
703 - } else {
704 - dmu_tx_abort(tx);
705 - }
687 +
688 + tx = dmu_tx_create(os);
689 + dmu_tx_hold_bonus(tx, object);
690 + dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
691 + err = dmu_tx_assign(tx, TXG_WAIT);
692 + if (err == 0) {
693 + err = dmu_object_free(os, object, tx);
694 + dmu_tx_commit(tx);
706 695 } else {
707 - err = dmu_free_long_range_impl(os, dn, 0, DMU_OBJECT_END, TRUE);
696 + dmu_tx_abort(tx);
708 697 }
709 - dnode_rele(dn, FTAG);
698 +
710 699 return (err);
711 700 }
712 701
713 702 int
714 703 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
715 704 uint64_t size, dmu_tx_t *tx)
716 705 {
717 706 dnode_t *dn;
718 707 int err = dnode_hold(os, object, FTAG, &dn);
719 708 if (err)
720 709 return (err);
721 710 ASSERT(offset < UINT64_MAX);
722 711 ASSERT(size == -1ULL || size <= UINT64_MAX - offset);
723 712 dnode_free_range(dn, offset, size, tx);
724 713 dnode_rele(dn, FTAG);
725 714 return (0);
726 715 }
727 716
728 717 int
729 718 dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
730 719 void *buf, uint32_t flags)
731 720 {
732 721 dnode_t *dn;
733 722 dmu_buf_t **dbp;
734 723 int numbufs, err;
735 724
736 725 err = dnode_hold(os, object, FTAG, &dn);
737 726 if (err)
738 727 return (err);
739 728
740 729 /*
741 730 * Deal with odd block sizes, where there can't be data past the first
742 731 * block. If we ever do the tail block optimization, we will need to
743 732 * handle that here as well.
744 733 */
745 734 if (dn->dn_maxblkid == 0) {
746 735 int newsz = offset > dn->dn_datablksz ? 0 :
747 736 MIN(size, dn->dn_datablksz - offset);
748 737 bzero((char *)buf + newsz, size - newsz);
749 738 size = newsz;
750 739 }
751 740
752 741 while (size > 0) {
753 742 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
754 743 int i;
755 744
756 745 /*
757 746 * NB: we could do this block-at-a-time, but it's nice
758 747 * to be reading in parallel.
759 748 */
760 749 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
761 750 TRUE, FTAG, &numbufs, &dbp, flags);
762 751 if (err)
763 752 break;
764 753
765 754 for (i = 0; i < numbufs; i++) {
766 755 int tocpy;
767 756 int bufoff;
768 757 dmu_buf_t *db = dbp[i];
769 758
770 759 ASSERT(size > 0);
771 760
772 761 bufoff = offset - db->db_offset;
773 762 tocpy = (int)MIN(db->db_size - bufoff, size);
774 763
775 764 bcopy((char *)db->db_data + bufoff, buf, tocpy);
776 765
777 766 offset += tocpy;
778 767 size -= tocpy;
779 768 buf = (char *)buf + tocpy;
780 769 }
781 770 dmu_buf_rele_array(dbp, numbufs, FTAG);
782 771 }
783 772 dnode_rele(dn, FTAG);
784 773 return (err);
785 774 }
786 775
787 776 void
788 777 dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
789 778 const void *buf, dmu_tx_t *tx)
790 779 {
791 780 dmu_buf_t **dbp;
792 781 int numbufs, i;
793 782
794 783 if (size == 0)
795 784 return;
796 785
797 786 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
798 787 FALSE, FTAG, &numbufs, &dbp));
799 788
800 789 for (i = 0; i < numbufs; i++) {
801 790 int tocpy;
802 791 int bufoff;
803 792 dmu_buf_t *db = dbp[i];
804 793
805 794 ASSERT(size > 0);
806 795
807 796 bufoff = offset - db->db_offset;
808 797 tocpy = (int)MIN(db->db_size - bufoff, size);
809 798
810 799 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
811 800
812 801 if (tocpy == db->db_size)
813 802 dmu_buf_will_fill(db, tx);
814 803 else
815 804 dmu_buf_will_dirty(db, tx);
816 805
817 806 bcopy(buf, (char *)db->db_data + bufoff, tocpy);
818 807
819 808 if (tocpy == db->db_size)
820 809 dmu_buf_fill_done(db, tx);
821 810
822 811 offset += tocpy;
823 812 size -= tocpy;
824 813 buf = (char *)buf + tocpy;
825 814 }
826 815 dmu_buf_rele_array(dbp, numbufs, FTAG);
827 816 }
828 817
829 818 void
830 819 dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
831 820 dmu_tx_t *tx)
832 821 {
833 822 dmu_buf_t **dbp;
834 823 int numbufs, i;
835 824
836 825 if (size == 0)
837 826 return;
838 827
839 828 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
840 829 FALSE, FTAG, &numbufs, &dbp));
841 830
842 831 for (i = 0; i < numbufs; i++) {
843 832 dmu_buf_t *db = dbp[i];
844 833
845 834 dmu_buf_will_not_fill(db, tx);
846 835 }
847 836 dmu_buf_rele_array(dbp, numbufs, FTAG);
848 837 }
849 838
850 839 /*
851 840 * DMU support for xuio
852 841 */
853 842 kstat_t *xuio_ksp = NULL;
854 843
855 844 int
856 845 dmu_xuio_init(xuio_t *xuio, int nblk)
857 846 {
858 847 dmu_xuio_t *priv;
859 848 uio_t *uio = &xuio->xu_uio;
860 849
861 850 uio->uio_iovcnt = nblk;
862 851 uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
863 852
864 853 priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
865 854 priv->cnt = nblk;
866 855 priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
867 856 priv->iovp = uio->uio_iov;
868 857 XUIO_XUZC_PRIV(xuio) = priv;
869 858
870 859 if (XUIO_XUZC_RW(xuio) == UIO_READ)
871 860 XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk);
872 861 else
873 862 XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk);
874 863
875 864 return (0);
876 865 }
877 866
878 867 void
879 868 dmu_xuio_fini(xuio_t *xuio)
880 869 {
881 870 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
882 871 int nblk = priv->cnt;
883 872
884 873 kmem_free(priv->iovp, nblk * sizeof (iovec_t));
885 874 kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *));
886 875 kmem_free(priv, sizeof (dmu_xuio_t));
887 876
888 877 if (XUIO_XUZC_RW(xuio) == UIO_READ)
889 878 XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk);
890 879 else
891 880 XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk);
892 881 }
893 882
894 883 /*
895 884 * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf }
896 885 * and increase priv->next by 1.
897 886 */
898 887 int
899 888 dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n)
900 889 {
901 890 struct iovec *iov;
902 891 uio_t *uio = &xuio->xu_uio;
903 892 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
904 893 int i = priv->next++;
905 894
906 895 ASSERT(i < priv->cnt);
907 896 ASSERT(off + n <= arc_buf_size(abuf));
908 897 iov = uio->uio_iov + i;
909 898 iov->iov_base = (char *)abuf->b_data + off;
910 899 iov->iov_len = n;
911 900 priv->bufs[i] = abuf;
912 901 return (0);
913 902 }
914 903
915 904 int
916 905 dmu_xuio_cnt(xuio_t *xuio)
917 906 {
918 907 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
919 908 return (priv->cnt);
920 909 }
921 910
922 911 arc_buf_t *
923 912 dmu_xuio_arcbuf(xuio_t *xuio, int i)
924 913 {
925 914 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
926 915
927 916 ASSERT(i < priv->cnt);
928 917 return (priv->bufs[i]);
929 918 }
930 919
931 920 void
932 921 dmu_xuio_clear(xuio_t *xuio, int i)
933 922 {
934 923 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
935 924
936 925 ASSERT(i < priv->cnt);
937 926 priv->bufs[i] = NULL;
938 927 }
939 928
940 929 static void
941 930 xuio_stat_init(void)
942 931 {
943 932 xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc",
944 933 KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t),
945 934 KSTAT_FLAG_VIRTUAL);
946 935 if (xuio_ksp != NULL) {
947 936 xuio_ksp->ks_data = &xuio_stats;
948 937 kstat_install(xuio_ksp);
949 938 }
950 939 }
951 940
952 941 static void
953 942 xuio_stat_fini(void)
954 943 {
955 944 if (xuio_ksp != NULL) {
956 945 kstat_delete(xuio_ksp);
957 946 xuio_ksp = NULL;
958 947 }
959 948 }
960 949
961 950 void
962 951 xuio_stat_wbuf_copied()
963 952 {
964 953 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
965 954 }
966 955
967 956 void
968 957 xuio_stat_wbuf_nocopy()
969 958 {
970 959 XUIOSTAT_BUMP(xuiostat_wbuf_nocopy);
971 960 }
972 961
973 962 #ifdef _KERNEL
974 963 int
975 964 dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
976 965 {
977 966 dmu_buf_t **dbp;
978 967 int numbufs, i, err;
979 968 xuio_t *xuio = NULL;
980 969
981 970 /*
982 971 * NB: we could do this block-at-a-time, but it's nice
983 972 * to be reading in parallel.
984 973 */
985 974 err = dmu_buf_hold_array(os, object, uio->uio_loffset, size, TRUE, FTAG,
986 975 &numbufs, &dbp);
987 976 if (err)
988 977 return (err);
989 978
990 979 if (uio->uio_extflg == UIO_XUIO)
991 980 xuio = (xuio_t *)uio;
992 981
993 982 for (i = 0; i < numbufs; i++) {
994 983 int tocpy;
995 984 int bufoff;
996 985 dmu_buf_t *db = dbp[i];
997 986
998 987 ASSERT(size > 0);
999 988
1000 989 bufoff = uio->uio_loffset - db->db_offset;
1001 990 tocpy = (int)MIN(db->db_size - bufoff, size);
1002 991
1003 992 if (xuio) {
1004 993 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
1005 994 arc_buf_t *dbuf_abuf = dbi->db_buf;
1006 995 arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
1007 996 err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
1008 997 if (!err) {
1009 998 uio->uio_resid -= tocpy;
1010 999 uio->uio_loffset += tocpy;
1011 1000 }
1012 1001
1013 1002 if (abuf == dbuf_abuf)
1014 1003 XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
1015 1004 else
1016 1005 XUIOSTAT_BUMP(xuiostat_rbuf_copied);
1017 1006 } else {
1018 1007 err = uiomove((char *)db->db_data + bufoff, tocpy,
1019 1008 UIO_READ, uio);
1020 1009 }
1021 1010 if (err)
1022 1011 break;
1023 1012
1024 1013 size -= tocpy;
1025 1014 }
1026 1015 dmu_buf_rele_array(dbp, numbufs, FTAG);
1027 1016
1028 1017 return (err);
1029 1018 }
1030 1019
1031 1020 static int
1032 1021 dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
1033 1022 {
1034 1023 dmu_buf_t **dbp;
1035 1024 int numbufs;
1036 1025 int err = 0;
1037 1026 int i;
1038 1027
1039 1028 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1040 1029 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1041 1030 if (err)
1042 1031 return (err);
1043 1032
1044 1033 for (i = 0; i < numbufs; i++) {
1045 1034 int tocpy;
1046 1035 int bufoff;
1047 1036 dmu_buf_t *db = dbp[i];
1048 1037
1049 1038 ASSERT(size > 0);
1050 1039
1051 1040 bufoff = uio->uio_loffset - db->db_offset;
1052 1041 tocpy = (int)MIN(db->db_size - bufoff, size);
1053 1042
1054 1043 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1055 1044
1056 1045 if (tocpy == db->db_size)
1057 1046 dmu_buf_will_fill(db, tx);
1058 1047 else
1059 1048 dmu_buf_will_dirty(db, tx);
1060 1049
1061 1050 /*
1062 1051 * XXX uiomove could block forever (eg. nfs-backed
1063 1052 * pages). There needs to be a uiolockdown() function
1064 1053 * to lock the pages in memory, so that uiomove won't
1065 1054 * block.
1066 1055 */
1067 1056 err = uiomove((char *)db->db_data + bufoff, tocpy,
1068 1057 UIO_WRITE, uio);
1069 1058
1070 1059 if (tocpy == db->db_size)
1071 1060 dmu_buf_fill_done(db, tx);
1072 1061
1073 1062 if (err)
1074 1063 break;
1075 1064
1076 1065 size -= tocpy;
1077 1066 }
1078 1067
1079 1068 dmu_buf_rele_array(dbp, numbufs, FTAG);
1080 1069 return (err);
1081 1070 }
1082 1071
1083 1072 int
1084 1073 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
1085 1074 dmu_tx_t *tx)
1086 1075 {
1087 1076 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1088 1077 dnode_t *dn;
1089 1078 int err;
1090 1079
1091 1080 if (size == 0)
1092 1081 return (0);
1093 1082
1094 1083 DB_DNODE_ENTER(db);
1095 1084 dn = DB_DNODE(db);
1096 1085 err = dmu_write_uio_dnode(dn, uio, size, tx);
1097 1086 DB_DNODE_EXIT(db);
1098 1087
1099 1088 return (err);
1100 1089 }
1101 1090
1102 1091 int
1103 1092 dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
1104 1093 dmu_tx_t *tx)
1105 1094 {
1106 1095 dnode_t *dn;
1107 1096 int err;
1108 1097
1109 1098 if (size == 0)
1110 1099 return (0);
1111 1100
1112 1101 err = dnode_hold(os, object, FTAG, &dn);
1113 1102 if (err)
1114 1103 return (err);
1115 1104
1116 1105 err = dmu_write_uio_dnode(dn, uio, size, tx);
1117 1106
1118 1107 dnode_rele(dn, FTAG);
1119 1108
1120 1109 return (err);
1121 1110 }
1122 1111
1123 1112 int
1124 1113 dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1125 1114 page_t *pp, dmu_tx_t *tx)
1126 1115 {
1127 1116 dmu_buf_t **dbp;
1128 1117 int numbufs, i;
1129 1118 int err;
1130 1119
1131 1120 if (size == 0)
1132 1121 return (0);
1133 1122
1134 1123 err = dmu_buf_hold_array(os, object, offset, size,
1135 1124 FALSE, FTAG, &numbufs, &dbp);
1136 1125 if (err)
1137 1126 return (err);
1138 1127
1139 1128 for (i = 0; i < numbufs; i++) {
1140 1129 int tocpy, copied, thiscpy;
1141 1130 int bufoff;
1142 1131 dmu_buf_t *db = dbp[i];
1143 1132 caddr_t va;
1144 1133
1145 1134 ASSERT(size > 0);
1146 1135 ASSERT3U(db->db_size, >=, PAGESIZE);
1147 1136
1148 1137 bufoff = offset - db->db_offset;
1149 1138 tocpy = (int)MIN(db->db_size - bufoff, size);
1150 1139
1151 1140 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1152 1141
1153 1142 if (tocpy == db->db_size)
1154 1143 dmu_buf_will_fill(db, tx);
1155 1144 else
1156 1145 dmu_buf_will_dirty(db, tx);
1157 1146
1158 1147 for (copied = 0; copied < tocpy; copied += PAGESIZE) {
1159 1148 ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff);
1160 1149 thiscpy = MIN(PAGESIZE, tocpy - copied);
1161 1150 va = zfs_map_page(pp, S_READ);
1162 1151 bcopy(va, (char *)db->db_data + bufoff, thiscpy);
1163 1152 zfs_unmap_page(pp, va);
1164 1153 pp = pp->p_next;
1165 1154 bufoff += PAGESIZE;
1166 1155 }
1167 1156
1168 1157 if (tocpy == db->db_size)
1169 1158 dmu_buf_fill_done(db, tx);
1170 1159
1171 1160 offset += tocpy;
1172 1161 size -= tocpy;
1173 1162 }
1174 1163 dmu_buf_rele_array(dbp, numbufs, FTAG);
1175 1164 return (err);
1176 1165 }
1177 1166 #endif
1178 1167
1179 1168 /*
1180 1169 * Allocate a loaned anonymous arc buffer.
1181 1170 */
1182 1171 arc_buf_t *
1183 1172 dmu_request_arcbuf(dmu_buf_t *handle, int size)
1184 1173 {
1185 1174 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
1186 1175 spa_t *spa;
1187 1176
1188 1177 DB_GET_SPA(&spa, db);
1189 1178 return (arc_loan_buf(spa, size));
1190 1179 }
1191 1180
1192 1181 /*
1193 1182 * Free a loaned arc buffer.
1194 1183 */
1195 1184 void
1196 1185 dmu_return_arcbuf(arc_buf_t *buf)
1197 1186 {
1198 1187 arc_return_buf(buf, FTAG);
1199 1188 VERIFY(arc_buf_remove_ref(buf, FTAG));
1200 1189 }
1201 1190
1202 1191 /*
1203 1192 * When possible directly assign passed loaned arc buffer to a dbuf.
1204 1193 * If this is not possible copy the contents of passed arc buf via
1205 1194 * dmu_write().
1206 1195 */
1207 1196 void
1208 1197 dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
1209 1198 dmu_tx_t *tx)
1210 1199 {
1211 1200 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1212 1201 dnode_t *dn;
1213 1202 dmu_buf_impl_t *db;
1214 1203 uint32_t blksz = (uint32_t)arc_buf_size(buf);
1215 1204 uint64_t blkid;
1216 1205
1217 1206 DB_DNODE_ENTER(dbuf);
1218 1207 dn = DB_DNODE(dbuf);
1219 1208 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1220 1209 blkid = dbuf_whichblock(dn, offset);
1221 1210 VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
1222 1211 rw_exit(&dn->dn_struct_rwlock);
1223 1212 DB_DNODE_EXIT(dbuf);
1224 1213
1225 1214 if (offset == db->db.db_offset && blksz == db->db.db_size) {
1226 1215 dbuf_assign_arcbuf(db, buf, tx);
1227 1216 dbuf_rele(db, FTAG);
1228 1217 } else {
1229 1218 objset_t *os;
1230 1219 uint64_t object;
1231 1220
1232 1221 DB_DNODE_ENTER(dbuf);
1233 1222 dn = DB_DNODE(dbuf);
1234 1223 os = dn->dn_objset;
1235 1224 object = dn->dn_object;
1236 1225 DB_DNODE_EXIT(dbuf);
1237 1226
1238 1227 dbuf_rele(db, FTAG);
1239 1228 dmu_write(os, object, offset, blksz, buf->b_data, tx);
1240 1229 dmu_return_arcbuf(buf);
1241 1230 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1242 1231 }
1243 1232 }
1244 1233
1245 1234 typedef struct {
1246 1235 dbuf_dirty_record_t *dsa_dr;
1247 1236 dmu_sync_cb_t *dsa_done;
1248 1237 zgd_t *dsa_zgd;
1249 1238 dmu_tx_t *dsa_tx;
1250 1239 } dmu_sync_arg_t;
1251 1240
1252 1241 /* ARGSUSED */
1253 1242 static void
1254 1243 dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1255 1244 {
1256 1245 dmu_sync_arg_t *dsa = varg;
1257 1246 dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1258 1247 blkptr_t *bp = zio->io_bp;
1259 1248
1260 1249 if (zio->io_error == 0) {
1261 1250 if (BP_IS_HOLE(bp)) {
1262 1251 /*
1263 1252 * A block of zeros may compress to a hole, but the
1264 1253 * block size still needs to be known for replay.
1265 1254 */
1266 1255 BP_SET_LSIZE(bp, db->db_size);
1267 1256 } else {
1268 1257 ASSERT(BP_GET_LEVEL(bp) == 0);
1269 1258 bp->blk_fill = 1;
1270 1259 }
1271 1260 }
1272 1261 }
1273 1262
1274 1263 static void
1275 1264 dmu_sync_late_arrival_ready(zio_t *zio)
1276 1265 {
1277 1266 dmu_sync_ready(zio, NULL, zio->io_private);
1278 1267 }
1279 1268
1280 1269 /* ARGSUSED */
1281 1270 static void
1282 1271 dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1283 1272 {
1284 1273 dmu_sync_arg_t *dsa = varg;
1285 1274 dbuf_dirty_record_t *dr = dsa->dsa_dr;
1286 1275 dmu_buf_impl_t *db = dr->dr_dbuf;
1287 1276
1288 1277 mutex_enter(&db->db_mtx);
1289 1278 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1290 1279 if (zio->io_error == 0) {
1291 1280 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
1292 1281 if (dr->dt.dl.dr_nopwrite) {
1293 1282 blkptr_t *bp = zio->io_bp;
1294 1283 blkptr_t *bp_orig = &zio->io_bp_orig;
1295 1284 uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
1296 1285
1297 1286 ASSERT(BP_EQUAL(bp, bp_orig));
1298 1287 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
1299 1288 ASSERT(zio_checksum_table[chksum].ci_dedup);
1300 1289 }
1301 1290 dr->dt.dl.dr_overridden_by = *zio->io_bp;
1302 1291 dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1303 1292 dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
1304 1293 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by))
1305 1294 BP_ZERO(&dr->dt.dl.dr_overridden_by);
1306 1295 } else {
1307 1296 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1308 1297 }
1309 1298 cv_broadcast(&db->db_changed);
1310 1299 mutex_exit(&db->db_mtx);
1311 1300
1312 1301 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1313 1302
1314 1303 kmem_free(dsa, sizeof (*dsa));
1315 1304 }
1316 1305
1317 1306 static void
1318 1307 dmu_sync_late_arrival_done(zio_t *zio)
1319 1308 {
1320 1309 blkptr_t *bp = zio->io_bp;
1321 1310 dmu_sync_arg_t *dsa = zio->io_private;
1322 1311 blkptr_t *bp_orig = &zio->io_bp_orig;
1323 1312
1324 1313 if (zio->io_error == 0 && !BP_IS_HOLE(bp)) {
1325 1314 /*
1326 1315 * If we didn't allocate a new block (i.e. ZIO_FLAG_NOPWRITE)
1327 1316 * then there is nothing to do here. Otherwise, free the
1328 1317 * newly allocated block in this txg.
1329 1318 */
1330 1319 if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
1331 1320 ASSERT(BP_EQUAL(bp, bp_orig));
1332 1321 } else {
1333 1322 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
1334 1323 ASSERT(zio->io_bp->blk_birth == zio->io_txg);
1335 1324 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
1336 1325 zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
1337 1326 }
1338 1327 }
1339 1328
1340 1329 dmu_tx_commit(dsa->dsa_tx);
1341 1330
1342 1331 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1343 1332
1344 1333 kmem_free(dsa, sizeof (*dsa));
1345 1334 }
1346 1335
1347 1336 static int
1348 1337 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
1349 1338 zio_prop_t *zp, zbookmark_t *zb)
1350 1339 {
1351 1340 dmu_sync_arg_t *dsa;
1352 1341 dmu_tx_t *tx;
1353 1342
1354 1343 tx = dmu_tx_create(os);
1355 1344 dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
1356 1345 if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1357 1346 dmu_tx_abort(tx);
1358 1347 /* Make zl_get_data do txg_waited_synced() */
1359 1348 return (SET_ERROR(EIO));
1360 1349 }
1361 1350
1362 1351 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1363 1352 dsa->dsa_dr = NULL;
1364 1353 dsa->dsa_done = done;
1365 1354 dsa->dsa_zgd = zgd;
1366 1355 dsa->dsa_tx = tx;
1367 1356
1368 1357 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
1369 1358 zgd->zgd_db->db_data, zgd->zgd_db->db_size, zp,
1370 1359 dmu_sync_late_arrival_ready, dmu_sync_late_arrival_done, dsa,
1371 1360 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
1372 1361
1373 1362 return (0);
1374 1363 }
1375 1364
1376 1365 /*
1377 1366 * Intent log support: sync the block associated with db to disk.
1378 1367 * N.B. and XXX: the caller is responsible for making sure that the
1379 1368 * data isn't changing while dmu_sync() is writing it.
1380 1369 *
1381 1370 * Return values:
1382 1371 *
1383 1372 * EEXIST: this txg has already been synced, so there's nothing to do.
1384 1373 * The caller should not log the write.
1385 1374 *
1386 1375 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
1387 1376 * The caller should not log the write.
1388 1377 *
1389 1378 * EALREADY: this block is already in the process of being synced.
1390 1379 * The caller should track its progress (somehow).
1391 1380 *
1392 1381 * EIO: could not do the I/O.
1393 1382 * The caller should do a txg_wait_synced().
1394 1383 *
1395 1384 * 0: the I/O has been initiated.
1396 1385 * The caller should log this blkptr in the done callback.
1397 1386 * It is possible that the I/O will fail, in which case
1398 1387 * the error will be reported to the done callback and
1399 1388 * propagated to pio from zio_done().
1400 1389 */
1401 1390 int
1402 1391 dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
1403 1392 {
1404 1393 blkptr_t *bp = zgd->zgd_bp;
1405 1394 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
1406 1395 objset_t *os = db->db_objset;
1407 1396 dsl_dataset_t *ds = os->os_dsl_dataset;
1408 1397 dbuf_dirty_record_t *dr;
1409 1398 dmu_sync_arg_t *dsa;
1410 1399 zbookmark_t zb;
1411 1400 zio_prop_t zp;
1412 1401 dnode_t *dn;
1413 1402
1414 1403 ASSERT(pio != NULL);
1415 1404 ASSERT(txg != 0);
1416 1405
1417 1406 SET_BOOKMARK(&zb, ds->ds_object,
1418 1407 db->db.db_object, db->db_level, db->db_blkid);
1419 1408
1420 1409 DB_DNODE_ENTER(db);
1421 1410 dn = DB_DNODE(db);
1422 1411 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
1423 1412 DB_DNODE_EXIT(db);
1424 1413
1425 1414 /*
1426 1415 * If we're frozen (running ziltest), we always need to generate a bp.
1427 1416 */
1428 1417 if (txg > spa_freeze_txg(os->os_spa))
1429 1418 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1430 1419
1431 1420 /*
1432 1421 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
1433 1422 * and us. If we determine that this txg is not yet syncing,
1434 1423 * but it begins to sync a moment later, that's OK because the
1435 1424 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
1436 1425 */
1437 1426 mutex_enter(&db->db_mtx);
1438 1427
1439 1428 if (txg <= spa_last_synced_txg(os->os_spa)) {
1440 1429 /*
1441 1430 * This txg has already synced. There's nothing to do.
1442 1431 */
1443 1432 mutex_exit(&db->db_mtx);
1444 1433 return (SET_ERROR(EEXIST));
1445 1434 }
1446 1435
1447 1436 if (txg <= spa_syncing_txg(os->os_spa)) {
1448 1437 /*
1449 1438 * This txg is currently syncing, so we can't mess with
1450 1439 * the dirty record anymore; just write a new log block.
1451 1440 */
1452 1441 mutex_exit(&db->db_mtx);
1453 1442 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1454 1443 }
1455 1444
1456 1445 dr = db->db_last_dirty;
1457 1446 while (dr && dr->dr_txg != txg)
1458 1447 dr = dr->dr_next;
1459 1448
1460 1449 if (dr == NULL) {
1461 1450 /*
1462 1451 * There's no dr for this dbuf, so it must have been freed.
1463 1452 * There's no need to log writes to freed blocks, so we're done.
1464 1453 */
1465 1454 mutex_exit(&db->db_mtx);
1466 1455 return (SET_ERROR(ENOENT));
1467 1456 }
1468 1457
1469 1458 ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
1470 1459
1471 1460 /*
1472 1461 * Assume the on-disk data is X, the current syncing data is Y,
1473 1462 * and the current in-memory data is Z (currently in dmu_sync).
1474 1463 * X and Z are identical but Y is has been modified. Normally,
1475 1464 * when X and Z are the same we will perform a nopwrite but if Y
1476 1465 * is different we must disable nopwrite since the resulting write
1477 1466 * of Y to disk can free the block containing X. If we allowed a
1478 1467 * nopwrite to occur the block pointing to Z would reference a freed
1479 1468 * block. Since this is a rare case we simplify this by disabling
1480 1469 * nopwrite if the current dmu_sync-ing dbuf has been modified in
1481 1470 * a previous transaction.
1482 1471 */
1483 1472 if (dr->dr_next)
1484 1473 zp.zp_nopwrite = B_FALSE;
1485 1474
1486 1475 ASSERT(dr->dr_txg == txg);
1487 1476 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
1488 1477 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1489 1478 /*
1490 1479 * We have already issued a sync write for this buffer,
1491 1480 * or this buffer has already been synced. It could not
1492 1481 * have been dirtied since, or we would have cleared the state.
1493 1482 */
1494 1483 mutex_exit(&db->db_mtx);
1495 1484 return (SET_ERROR(EALREADY));
1496 1485 }
1497 1486
1498 1487 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
1499 1488 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
1500 1489 mutex_exit(&db->db_mtx);
1501 1490
1502 1491 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1503 1492 dsa->dsa_dr = dr;
1504 1493 dsa->dsa_done = done;
1505 1494 dsa->dsa_zgd = zgd;
1506 1495 dsa->dsa_tx = NULL;
1507 1496
1508 1497 zio_nowait(arc_write(pio, os->os_spa, txg,
1509 1498 bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
1510 1499 DBUF_IS_L2COMPRESSIBLE(db), &zp, dmu_sync_ready, dmu_sync_done,
1511 1500 dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
1512 1501
1513 1502 return (0);
1514 1503 }
1515 1504
1516 1505 int
1517 1506 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
1518 1507 dmu_tx_t *tx)
1519 1508 {
1520 1509 dnode_t *dn;
1521 1510 int err;
1522 1511
1523 1512 err = dnode_hold(os, object, FTAG, &dn);
1524 1513 if (err)
1525 1514 return (err);
1526 1515 err = dnode_set_blksz(dn, size, ibs, tx);
1527 1516 dnode_rele(dn, FTAG);
1528 1517 return (err);
1529 1518 }
1530 1519
1531 1520 void
1532 1521 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
1533 1522 dmu_tx_t *tx)
1534 1523 {
1535 1524 dnode_t *dn;
1536 1525
1537 1526 /* XXX assumes dnode_hold will not get an i/o error */
1538 1527 (void) dnode_hold(os, object, FTAG, &dn);
1539 1528 ASSERT(checksum < ZIO_CHECKSUM_FUNCTIONS);
1540 1529 dn->dn_checksum = checksum;
1541 1530 dnode_setdirty(dn, tx);
1542 1531 dnode_rele(dn, FTAG);
1543 1532 }
1544 1533
1545 1534 void
1546 1535 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
1547 1536 dmu_tx_t *tx)
1548 1537 {
1549 1538 dnode_t *dn;
1550 1539
1551 1540 /* XXX assumes dnode_hold will not get an i/o error */
1552 1541 (void) dnode_hold(os, object, FTAG, &dn);
1553 1542 ASSERT(compress < ZIO_COMPRESS_FUNCTIONS);
1554 1543 dn->dn_compress = compress;
1555 1544 dnode_setdirty(dn, tx);
1556 1545 dnode_rele(dn, FTAG);
1557 1546 }
1558 1547
1559 1548 int zfs_mdcomp_disable = 0;
1560 1549
1561 1550 void
1562 1551 dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
1563 1552 {
1564 1553 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
1565 1554 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
1566 1555 (wp & WP_SPILL));
1567 1556 enum zio_checksum checksum = os->os_checksum;
1568 1557 enum zio_compress compress = os->os_compress;
1569 1558 enum zio_checksum dedup_checksum = os->os_dedup_checksum;
1570 1559 boolean_t dedup = B_FALSE;
1571 1560 boolean_t nopwrite = B_FALSE;
1572 1561 boolean_t dedup_verify = os->os_dedup_verify;
1573 1562 int copies = os->os_copies;
1574 1563
1575 1564 /*
1576 1565 * We maintain different write policies for each of the following
1577 1566 * types of data:
1578 1567 * 1. metadata
1579 1568 * 2. preallocated blocks (i.e. level-0 blocks of a dump device)
1580 1569 * 3. all other level 0 blocks
1581 1570 */
1582 1571 if (ismd) {
1583 1572 /*
1584 1573 * XXX -- we should design a compression algorithm
1585 1574 * that specializes in arrays of bps.
1586 1575 */
1587 1576 compress = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY :
1588 1577 ZIO_COMPRESS_LZJB;
1589 1578
1590 1579 /*
1591 1580 * Metadata always gets checksummed. If the data
1592 1581 * checksum is multi-bit correctable, and it's not a
1593 1582 * ZBT-style checksum, then it's suitable for metadata
1594 1583 * as well. Otherwise, the metadata checksum defaults
1595 1584 * to fletcher4.
1596 1585 */
1597 1586 if (zio_checksum_table[checksum].ci_correctable < 1 ||
1598 1587 zio_checksum_table[checksum].ci_eck)
1599 1588 checksum = ZIO_CHECKSUM_FLETCHER_4;
1600 1589 } else if (wp & WP_NOFILL) {
1601 1590 ASSERT(level == 0);
1602 1591
1603 1592 /*
1604 1593 * If we're writing preallocated blocks, we aren't actually
1605 1594 * writing them so don't set any policy properties. These
1606 1595 * blocks are currently only used by an external subsystem
1607 1596 * outside of zfs (i.e. dump) and not written by the zio
1608 1597 * pipeline.
1609 1598 */
1610 1599 compress = ZIO_COMPRESS_OFF;
1611 1600 checksum = ZIO_CHECKSUM_OFF;
1612 1601 } else {
1613 1602 compress = zio_compress_select(dn->dn_compress, compress);
1614 1603
1615 1604 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
1616 1605 zio_checksum_select(dn->dn_checksum, checksum) :
1617 1606 dedup_checksum;
1618 1607
1619 1608 /*
1620 1609 * Determine dedup setting. If we are in dmu_sync(),
1621 1610 * we won't actually dedup now because that's all
1622 1611 * done in syncing context; but we do want to use the
1623 1612 * dedup checkum. If the checksum is not strong
1624 1613 * enough to ensure unique signatures, force
1625 1614 * dedup_verify.
1626 1615 */
1627 1616 if (dedup_checksum != ZIO_CHECKSUM_OFF) {
1628 1617 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
1629 1618 if (!zio_checksum_table[checksum].ci_dedup)
1630 1619 dedup_verify = B_TRUE;
1631 1620 }
1632 1621
1633 1622 /*
1634 1623 * Enable nopwrite if we have a cryptographically secure
1635 1624 * checksum that has no known collisions (i.e. SHA-256)
1636 1625 * and compression is enabled. We don't enable nopwrite if
1637 1626 * dedup is enabled as the two features are mutually exclusive.
1638 1627 */
1639 1628 nopwrite = (!dedup && zio_checksum_table[checksum].ci_dedup &&
1640 1629 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
1641 1630 }
1642 1631
1643 1632 zp->zp_checksum = checksum;
1644 1633 zp->zp_compress = compress;
1645 1634 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
1646 1635 zp->zp_level = level;
1647 1636 zp->zp_copies = MIN(copies + ismd, spa_max_replication(os->os_spa));
1648 1637 zp->zp_dedup = dedup;
1649 1638 zp->zp_dedup_verify = dedup && dedup_verify;
1650 1639 zp->zp_nopwrite = nopwrite;
1651 1640 }
1652 1641
1653 1642 int
1654 1643 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
1655 1644 {
1656 1645 dnode_t *dn;
1657 1646 int i, err;
1658 1647
1659 1648 err = dnode_hold(os, object, FTAG, &dn);
1660 1649 if (err)
1661 1650 return (err);
1662 1651 /*
1663 1652 * Sync any current changes before
1664 1653 * we go trundling through the block pointers.
1665 1654 */
1666 1655 for (i = 0; i < TXG_SIZE; i++) {
1667 1656 if (list_link_active(&dn->dn_dirty_link[i]))
1668 1657 break;
1669 1658 }
1670 1659 if (i != TXG_SIZE) {
1671 1660 dnode_rele(dn, FTAG);
1672 1661 txg_wait_synced(dmu_objset_pool(os), 0);
1673 1662 err = dnode_hold(os, object, FTAG, &dn);
1674 1663 if (err)
1675 1664 return (err);
1676 1665 }
1677 1666
1678 1667 err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
1679 1668 dnode_rele(dn, FTAG);
1680 1669
1681 1670 return (err);
1682 1671 }
1683 1672
1684 1673 void
1685 1674 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
1686 1675 {
1687 1676 dnode_phys_t *dnp;
1688 1677
1689 1678 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1690 1679 mutex_enter(&dn->dn_mtx);
1691 1680
1692 1681 dnp = dn->dn_phys;
1693 1682
1694 1683 doi->doi_data_block_size = dn->dn_datablksz;
1695 1684 doi->doi_metadata_block_size = dn->dn_indblkshift ?
1696 1685 1ULL << dn->dn_indblkshift : 0;
1697 1686 doi->doi_type = dn->dn_type;
1698 1687 doi->doi_bonus_type = dn->dn_bonustype;
1699 1688 doi->doi_bonus_size = dn->dn_bonuslen;
1700 1689 doi->doi_indirection = dn->dn_nlevels;
1701 1690 doi->doi_checksum = dn->dn_checksum;
1702 1691 doi->doi_compress = dn->dn_compress;
1703 1692 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
1704 1693 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
1705 1694 doi->doi_fill_count = 0;
1706 1695 for (int i = 0; i < dnp->dn_nblkptr; i++)
1707 1696 doi->doi_fill_count += dnp->dn_blkptr[i].blk_fill;
1708 1697
1709 1698 mutex_exit(&dn->dn_mtx);
1710 1699 rw_exit(&dn->dn_struct_rwlock);
1711 1700 }
1712 1701
1713 1702 /*
1714 1703 * Get information on a DMU object.
1715 1704 * If doi is NULL, just indicates whether the object exists.
1716 1705 */
1717 1706 int
1718 1707 dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
1719 1708 {
1720 1709 dnode_t *dn;
1721 1710 int err = dnode_hold(os, object, FTAG, &dn);
1722 1711
1723 1712 if (err)
1724 1713 return (err);
1725 1714
1726 1715 if (doi != NULL)
1727 1716 dmu_object_info_from_dnode(dn, doi);
1728 1717
1729 1718 dnode_rele(dn, FTAG);
1730 1719 return (0);
1731 1720 }
1732 1721
1733 1722 /*
1734 1723 * As above, but faster; can be used when you have a held dbuf in hand.
1735 1724 */
1736 1725 void
1737 1726 dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
1738 1727 {
1739 1728 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1740 1729
1741 1730 DB_DNODE_ENTER(db);
1742 1731 dmu_object_info_from_dnode(DB_DNODE(db), doi);
1743 1732 DB_DNODE_EXIT(db);
1744 1733 }
1745 1734
1746 1735 /*
1747 1736 * Faster still when you only care about the size.
1748 1737 * This is specifically optimized for zfs_getattr().
1749 1738 */
1750 1739 void
1751 1740 dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
1752 1741 u_longlong_t *nblk512)
1753 1742 {
1754 1743 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1755 1744 dnode_t *dn;
1756 1745
1757 1746 DB_DNODE_ENTER(db);
1758 1747 dn = DB_DNODE(db);
1759 1748
1760 1749 *blksize = dn->dn_datablksz;
1761 1750 /* add 1 for dnode space */
1762 1751 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
1763 1752 SPA_MINBLOCKSHIFT) + 1;
1764 1753 DB_DNODE_EXIT(db);
1765 1754 }
1766 1755
1767 1756 void
1768 1757 byteswap_uint64_array(void *vbuf, size_t size)
1769 1758 {
1770 1759 uint64_t *buf = vbuf;
1771 1760 size_t count = size >> 3;
1772 1761 int i;
1773 1762
1774 1763 ASSERT((size & 7) == 0);
1775 1764
1776 1765 for (i = 0; i < count; i++)
1777 1766 buf[i] = BSWAP_64(buf[i]);
1778 1767 }
1779 1768
1780 1769 void
1781 1770 byteswap_uint32_array(void *vbuf, size_t size)
1782 1771 {
1783 1772 uint32_t *buf = vbuf;
1784 1773 size_t count = size >> 2;
1785 1774 int i;
1786 1775
1787 1776 ASSERT((size & 3) == 0);
1788 1777
1789 1778 for (i = 0; i < count; i++)
1790 1779 buf[i] = BSWAP_32(buf[i]);
1791 1780 }
1792 1781
1793 1782 void
1794 1783 byteswap_uint16_array(void *vbuf, size_t size)
1795 1784 {
1796 1785 uint16_t *buf = vbuf;
1797 1786 size_t count = size >> 1;
1798 1787 int i;
1799 1788
1800 1789 ASSERT((size & 1) == 0);
1801 1790
1802 1791 for (i = 0; i < count; i++)
1803 1792 buf[i] = BSWAP_16(buf[i]);
1804 1793 }
1805 1794
1806 1795 /* ARGSUSED */
1807 1796 void
1808 1797 byteswap_uint8_array(void *vbuf, size_t size)
1809 1798 {
1810 1799 }
1811 1800
1812 1801 void
1813 1802 dmu_init(void)
1814 1803 {
1815 1804 zfs_dbgmsg_init();
1816 1805 sa_cache_init();
1817 1806 xuio_stat_init();
1818 1807 dmu_objset_init();
1819 1808 dnode_init();
1820 1809 dbuf_init();
1821 1810 zfetch_init();
1822 1811 l2arc_init();
1823 1812 arc_init();
1824 1813 }
1825 1814
1826 1815 void
1827 1816 dmu_fini(void)
1828 1817 {
1829 1818 arc_fini(); /* arc depends on l2arc, so arc must go first */
1830 1819 l2arc_fini();
1831 1820 zfetch_fini();
1832 1821 dbuf_fini();
1833 1822 dnode_fini();
1834 1823 dmu_objset_fini();
1835 1824 xuio_stat_fini();
1836 1825 sa_cache_fini();
1837 1826 zfs_dbgmsg_fini();
1838 1827 }
|
↓ open down ↓ |
1119 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX