Print this page
11421 smatch noise in dmu.c
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/dmu.c
+++ new/usr/src/uts/common/fs/zfs/dmu.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
|
↓ open down ↓ |
15 lines elided |
↑ open up ↑ |
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24 /*
25 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
26 - * Copyright (c) 2013, Joyent, Inc. All rights reserved.
26 + * Copyright 2019 Joyent, Inc.
27 27 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
28 28 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
29 29 * Copyright (c) 2018 DilOS
30 30 */
31 31
32 32 #include <sys/dmu.h>
33 33 #include <sys/dmu_impl.h>
34 34 #include <sys/dmu_tx.h>
35 35 #include <sys/dbuf.h>
36 36 #include <sys/dnode.h>
37 37 #include <sys/zfs_context.h>
38 38 #include <sys/dmu_objset.h>
39 39 #include <sys/dmu_traverse.h>
40 40 #include <sys/dsl_dataset.h>
41 41 #include <sys/dsl_dir.h>
42 42 #include <sys/dsl_pool.h>
43 43 #include <sys/dsl_synctask.h>
44 44 #include <sys/dsl_prop.h>
45 45 #include <sys/dmu_zfetch.h>
46 46 #include <sys/zfs_ioctl.h>
47 47 #include <sys/zap.h>
48 48 #include <sys/zio_checksum.h>
49 49 #include <sys/zio_compress.h>
50 50 #include <sys/sa.h>
51 51 #include <sys/zfeature.h>
52 52 #include <sys/abd.h>
53 53 #ifdef _KERNEL
54 54 #include <sys/vmsystm.h>
55 55 #include <sys/zfs_znode.h>
56 56 #endif
57 57
58 58 static xuio_stats_t xuio_stats = {
59 59 { "onloan_read_buf", KSTAT_DATA_UINT64 },
60 60 { "onloan_write_buf", KSTAT_DATA_UINT64 },
61 61 { "read_buf_copied", KSTAT_DATA_UINT64 },
62 62 { "read_buf_nocopy", KSTAT_DATA_UINT64 },
63 63 { "write_buf_copied", KSTAT_DATA_UINT64 },
64 64 { "write_buf_nocopy", KSTAT_DATA_UINT64 }
65 65 };
66 66
67 67 #define XUIOSTAT_INCR(stat, val) \
68 68 atomic_add_64(&xuio_stats.stat.value.ui64, (val))
69 69 #define XUIOSTAT_BUMP(stat) XUIOSTAT_INCR(stat, 1)
70 70
71 71 /*
72 72 * Enable/disable nopwrite feature.
73 73 */
74 74 int zfs_nopwrite_enabled = 1;
75 75
76 76 /*
77 77 * Tunable to control percentage of dirtied blocks from frees in one TXG.
78 78 * After this threshold is crossed, additional dirty blocks from frees
79 79 * wait until the next TXG.
80 80 * A value of zero will disable this throttle.
81 81 */
82 82 uint32_t zfs_per_txg_dirty_frees_percent = 30;
83 83
84 84 /*
85 85 * This can be used for testing, to ensure that certain actions happen
86 86 * while in the middle of a remap (which might otherwise complete too
87 87 * quickly).
88 88 */
89 89 int zfs_object_remap_one_indirect_delay_ticks = 0;
90 90
91 91 /*
92 92 * Limit the amount we can prefetch with one call to this amount. This
93 93 * helps to limit the amount of memory that can be used by prefetching.
94 94 * Larger objects should be prefetched a bit at a time.
95 95 */
96 96 uint64_t dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE;
97 97
98 98 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
99 99 { DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" },
100 100 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" },
101 101 { DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" },
102 102 { DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" },
103 103 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" },
104 104 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" },
105 105 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" },
106 106 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" },
107 107 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" },
108 108 { DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" },
109 109 { DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" },
110 110 { DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" },
111 111 { DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" },
112 112 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map" },
113 113 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" },
114 114 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" },
115 115 { DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" },
116 116 { DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" },
117 117 { DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" },
118 118 { DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" },
119 119 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" },
120 120 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" },
121 121 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" },
122 122 { DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" },
123 123 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" },
124 124 { DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" },
125 125 { DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" },
126 126 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" },
127 127 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" },
128 128 { DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" },
129 129 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" },
130 130 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" },
131 131 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" },
132 132 { DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" },
133 133 { DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" },
134 134 { DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" },
135 135 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" },
136 136 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones" },
137 137 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" },
138 138 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group used" },
139 139 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group quota" },
140 140 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags" },
141 141 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" },
142 142 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" },
143 143 { DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" },
144 144 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" },
145 145 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" },
146 146 { DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" },
147 147 { DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" },
148 148 { DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" },
149 149 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" },
150 150 { DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" },
151 151 { DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" },
152 152 { DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" }
153 153 };
154 154
155 155 const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
156 156 { byteswap_uint8_array, "uint8" },
157 157 { byteswap_uint16_array, "uint16" },
158 158 { byteswap_uint32_array, "uint32" },
159 159 { byteswap_uint64_array, "uint64" },
160 160 { zap_byteswap, "zap" },
161 161 { dnode_buf_byteswap, "dnode" },
162 162 { dmu_objset_byteswap, "objset" },
163 163 { zfs_znode_byteswap, "znode" },
164 164 { zfs_oldacl_byteswap, "oldacl" },
165 165 { zfs_acl_byteswap, "acl" }
166 166 };
167 167
168 168 int
169 169 dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset,
170 170 void *tag, dmu_buf_t **dbp)
171 171 {
172 172 uint64_t blkid;
173 173 dmu_buf_impl_t *db;
174 174
175 175 blkid = dbuf_whichblock(dn, 0, offset);
176 176 rw_enter(&dn->dn_struct_rwlock, RW_READER);
177 177 db = dbuf_hold(dn, blkid, tag);
178 178 rw_exit(&dn->dn_struct_rwlock);
179 179
180 180 if (db == NULL) {
181 181 *dbp = NULL;
182 182 return (SET_ERROR(EIO));
183 183 }
184 184
185 185 *dbp = &db->db;
186 186 return (0);
187 187 }
188 188 int
189 189 dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
190 190 void *tag, dmu_buf_t **dbp)
191 191 {
192 192 dnode_t *dn;
193 193 uint64_t blkid;
194 194 dmu_buf_impl_t *db;
195 195 int err;
196 196
197 197 err = dnode_hold(os, object, FTAG, &dn);
198 198 if (err)
199 199 return (err);
200 200 blkid = dbuf_whichblock(dn, 0, offset);
201 201 rw_enter(&dn->dn_struct_rwlock, RW_READER);
202 202 db = dbuf_hold(dn, blkid, tag);
203 203 rw_exit(&dn->dn_struct_rwlock);
204 204 dnode_rele(dn, FTAG);
205 205
206 206 if (db == NULL) {
207 207 *dbp = NULL;
208 208 return (SET_ERROR(EIO));
209 209 }
210 210
211 211 *dbp = &db->db;
212 212 return (err);
213 213 }
214 214
215 215 int
216 216 dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
217 217 void *tag, dmu_buf_t **dbp, int flags)
218 218 {
219 219 int err;
220 220 int db_flags = DB_RF_CANFAIL;
221 221
222 222 if (flags & DMU_READ_NO_PREFETCH)
223 223 db_flags |= DB_RF_NOPREFETCH;
224 224 if (flags & DMU_READ_NO_DECRYPT)
225 225 db_flags |= DB_RF_NO_DECRYPT;
226 226
227 227 err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp);
228 228 if (err == 0) {
229 229 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
230 230 err = dbuf_read(db, NULL, db_flags);
231 231 if (err != 0) {
232 232 dbuf_rele(db, tag);
233 233 *dbp = NULL;
234 234 }
235 235 }
236 236
237 237 return (err);
238 238 }
239 239
240 240 int
241 241 dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
242 242 void *tag, dmu_buf_t **dbp, int flags)
243 243 {
244 244 int err;
245 245 int db_flags = DB_RF_CANFAIL;
246 246
247 247 if (flags & DMU_READ_NO_PREFETCH)
248 248 db_flags |= DB_RF_NOPREFETCH;
249 249 if (flags & DMU_READ_NO_DECRYPT)
250 250 db_flags |= DB_RF_NO_DECRYPT;
251 251
252 252 err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
253 253 if (err == 0) {
254 254 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
255 255 err = dbuf_read(db, NULL, db_flags);
256 256 if (err != 0) {
257 257 dbuf_rele(db, tag);
258 258 *dbp = NULL;
259 259 }
260 260 }
261 261
262 262 return (err);
263 263 }
264 264
265 265 int
266 266 dmu_bonus_max(void)
267 267 {
268 268 return (DN_OLD_MAX_BONUSLEN);
269 269 }
270 270
271 271 int
272 272 dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
273 273 {
274 274 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
275 275 dnode_t *dn;
276 276 int error;
277 277
278 278 DB_DNODE_ENTER(db);
279 279 dn = DB_DNODE(db);
280 280
281 281 if (dn->dn_bonus != db) {
282 282 error = SET_ERROR(EINVAL);
283 283 } else if (newsize < 0 || newsize > db_fake->db_size) {
284 284 error = SET_ERROR(EINVAL);
285 285 } else {
286 286 dnode_setbonuslen(dn, newsize, tx);
287 287 error = 0;
288 288 }
289 289
290 290 DB_DNODE_EXIT(db);
291 291 return (error);
292 292 }
293 293
294 294 int
295 295 dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
296 296 {
297 297 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
298 298 dnode_t *dn;
299 299 int error;
300 300
301 301 DB_DNODE_ENTER(db);
302 302 dn = DB_DNODE(db);
303 303
304 304 if (!DMU_OT_IS_VALID(type)) {
305 305 error = SET_ERROR(EINVAL);
306 306 } else if (dn->dn_bonus != db) {
307 307 error = SET_ERROR(EINVAL);
308 308 } else {
309 309 dnode_setbonus_type(dn, type, tx);
310 310 error = 0;
311 311 }
312 312
313 313 DB_DNODE_EXIT(db);
314 314 return (error);
315 315 }
316 316
317 317 dmu_object_type_t
318 318 dmu_get_bonustype(dmu_buf_t *db_fake)
319 319 {
320 320 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
321 321 dnode_t *dn;
322 322 dmu_object_type_t type;
323 323
324 324 DB_DNODE_ENTER(db);
325 325 dn = DB_DNODE(db);
326 326 type = dn->dn_bonustype;
327 327 DB_DNODE_EXIT(db);
328 328
329 329 return (type);
330 330 }
331 331
332 332 int
333 333 dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
334 334 {
335 335 dnode_t *dn;
336 336 int error;
337 337
338 338 error = dnode_hold(os, object, FTAG, &dn);
339 339 dbuf_rm_spill(dn, tx);
340 340 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
341 341 dnode_rm_spill(dn, tx);
342 342 rw_exit(&dn->dn_struct_rwlock);
343 343 dnode_rele(dn, FTAG);
344 344 return (error);
345 345 }
346 346
347 347 /*
348 348 * Lookup and hold the bonus buffer for the provided dnode. If the dnode
349 349 * has not yet been allocated a new bonus dbuf a will be allocated.
350 350 * Returns ENOENT, EIO, or 0.
351 351 */
|
↓ open down ↓ |
315 lines elided |
↑ open up ↑ |
352 352 int dmu_bonus_hold_by_dnode(dnode_t *dn, void *tag, dmu_buf_t **dbp,
353 353 uint32_t flags)
354 354 {
355 355 dmu_buf_impl_t *db;
356 356 int error;
357 357 uint32_t db_flags = DB_RF_MUST_SUCCEED;
358 358
359 359 if (flags & DMU_READ_NO_PREFETCH)
360 360 db_flags |= DB_RF_NOPREFETCH;
361 361 if (flags & DMU_READ_NO_DECRYPT)
362 - db_flags |= DB_RF_NO_DECRYPT;
362 + db_flags |= DB_RF_NO_DECRYPT;
363 363
364 364 rw_enter(&dn->dn_struct_rwlock, RW_READER);
365 365 if (dn->dn_bonus == NULL) {
366 366 rw_exit(&dn->dn_struct_rwlock);
367 367 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
368 368 if (dn->dn_bonus == NULL)
369 369 dbuf_create_bonus(dn);
370 370 }
371 371 db = dn->dn_bonus;
372 372
373 373 /* as long as the bonus buf is held, the dnode will be held */
374 374 if (zfs_refcount_add(&db->db_holds, tag) == 1) {
375 375 VERIFY(dnode_add_ref(dn, db));
376 376 atomic_inc_32(&dn->dn_dbufs_count);
377 377 }
378 378
379 379 /*
380 380 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
381 381 * hold and incrementing the dbuf count to ensure that dnode_move() sees
382 382 * a dnode hold for every dbuf.
383 383 */
384 384 rw_exit(&dn->dn_struct_rwlock);
385 385
386 386 error = dbuf_read(db, NULL, db_flags);
387 387 if (error) {
388 388 dnode_evict_bonus(dn);
389 389 dbuf_rele(db, tag);
390 390 *dbp = NULL;
391 391 return (error);
392 392 }
393 393
394 394 *dbp = &db->db;
395 395 return (0);
396 396 }
397 397
398 398 /*
399 399 * returns ENOENT, EIO, or 0.
400 400 */
401 401 int
402 402 dmu_bonus_hold_impl(objset_t *os, uint64_t object, void *tag, uint32_t flags,
403 403 dmu_buf_t **dbp)
404 404 {
405 405 dnode_t *dn;
406 406 dmu_buf_impl_t *db;
407 407 int error;
408 408 uint32_t db_flags = DB_RF_MUST_SUCCEED;
409 409
410 410 if (flags & DMU_READ_NO_PREFETCH)
411 411 db_flags |= DB_RF_NOPREFETCH;
412 412 if (flags & DMU_READ_NO_DECRYPT)
413 413 db_flags |= DB_RF_NO_DECRYPT;
414 414
415 415 error = dnode_hold(os, object, FTAG, &dn);
416 416 if (error)
417 417 return (error);
418 418
419 419 rw_enter(&dn->dn_struct_rwlock, RW_READER);
420 420 if (dn->dn_bonus == NULL) {
421 421 rw_exit(&dn->dn_struct_rwlock);
422 422 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
423 423 if (dn->dn_bonus == NULL)
424 424 dbuf_create_bonus(dn);
425 425 }
426 426 db = dn->dn_bonus;
427 427
428 428 /* as long as the bonus buf is held, the dnode will be held */
429 429 if (zfs_refcount_add(&db->db_holds, tag) == 1) {
430 430 VERIFY(dnode_add_ref(dn, db));
431 431 atomic_inc_32(&dn->dn_dbufs_count);
432 432 }
433 433
434 434 /*
435 435 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
436 436 * hold and incrementing the dbuf count to ensure that dnode_move() sees
437 437 * a dnode hold for every dbuf.
438 438 */
439 439 rw_exit(&dn->dn_struct_rwlock);
440 440
441 441 dnode_rele(dn, FTAG);
442 442
443 443 error = dbuf_read(db, NULL, db_flags);
444 444 if (error) {
445 445 dnode_evict_bonus(dn);
446 446 dbuf_rele(db, tag);
447 447 *dbp = NULL;
448 448 return (error);
449 449 }
450 450
451 451 *dbp = &db->db;
452 452 return (0);
453 453 }
454 454
455 455 int
456 456 dmu_bonus_hold(objset_t *os, uint64_t obj, void *tag, dmu_buf_t **dbp)
457 457 {
458 458 return (dmu_bonus_hold_impl(os, obj, tag, DMU_READ_NO_PREFETCH, dbp));
459 459 }
460 460
461 461 /*
462 462 * returns ENOENT, EIO, or 0.
463 463 *
464 464 * This interface will allocate a blank spill dbuf when a spill blk
465 465 * doesn't already exist on the dnode.
466 466 *
467 467 * if you only want to find an already existing spill db, then
468 468 * dmu_spill_hold_existing() should be used.
469 469 */
470 470 int
471 471 dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
472 472 {
473 473 dmu_buf_impl_t *db = NULL;
474 474 int err;
475 475
476 476 if ((flags & DB_RF_HAVESTRUCT) == 0)
477 477 rw_enter(&dn->dn_struct_rwlock, RW_READER);
478 478
479 479 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
480 480
481 481 if ((flags & DB_RF_HAVESTRUCT) == 0)
482 482 rw_exit(&dn->dn_struct_rwlock);
483 483
484 484 ASSERT(db != NULL);
485 485 err = dbuf_read(db, NULL, flags);
486 486 if (err == 0)
487 487 *dbp = &db->db;
488 488 else
489 489 dbuf_rele(db, tag);
490 490 return (err);
491 491 }
492 492
493 493 int
494 494 dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
495 495 {
496 496 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
497 497 dnode_t *dn;
498 498 int err;
499 499
500 500 DB_DNODE_ENTER(db);
501 501 dn = DB_DNODE(db);
502 502
503 503 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
504 504 err = SET_ERROR(EINVAL);
505 505 } else {
506 506 rw_enter(&dn->dn_struct_rwlock, RW_READER);
507 507
508 508 if (!dn->dn_have_spill) {
509 509 err = SET_ERROR(ENOENT);
510 510 } else {
511 511 err = dmu_spill_hold_by_dnode(dn,
512 512 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
513 513 }
514 514
515 515 rw_exit(&dn->dn_struct_rwlock);
516 516 }
517 517
518 518 DB_DNODE_EXIT(db);
519 519 return (err);
520 520 }
521 521
522 522 int
523 523 dmu_spill_hold_by_bonus(dmu_buf_t *bonus, uint32_t flags, void *tag,
524 524 dmu_buf_t **dbp)
525 525 {
526 526 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
527 527 dnode_t *dn;
528 528 int err;
529 529 uint32_t db_flags = DB_RF_CANFAIL;
530 530
531 531 if (flags & DMU_READ_NO_DECRYPT)
532 532 db_flags |= DB_RF_NO_DECRYPT;
533 533
534 534 DB_DNODE_ENTER(db);
535 535 dn = DB_DNODE(db);
536 536 err = dmu_spill_hold_by_dnode(dn, db_flags, tag, dbp);
537 537 DB_DNODE_EXIT(db);
538 538
539 539 return (err);
540 540 }
541 541
542 542 /*
543 543 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
544 544 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
545 545 * and can induce severe lock contention when writing to several files
546 546 * whose dnodes are in the same block.
547 547 */
548 548 int
549 549 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
550 550 boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
551 551 {
552 552 dmu_buf_t **dbp;
553 553 uint64_t blkid, nblks, i;
554 554 uint32_t dbuf_flags;
555 555 int err;
556 556 zio_t *zio;
557 557
558 558 ASSERT(length <= DMU_MAX_ACCESS);
559 559
560 560 /*
561 561 * Note: We directly notify the prefetch code of this read, so that
562 562 * we can tell it about the multi-block read. dbuf_read() only knows
563 563 * about the one block it is accessing.
564 564 */
565 565 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT |
566 566 DB_RF_NOPREFETCH;
567 567
568 568 rw_enter(&dn->dn_struct_rwlock, RW_READER);
569 569 if (dn->dn_datablkshift) {
570 570 int blkshift = dn->dn_datablkshift;
571 571 nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
572 572 P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
573 573 } else {
574 574 if (offset + length > dn->dn_datablksz) {
575 575 zfs_panic_recover("zfs: accessing past end of object "
576 576 "%llx/%llx (size=%u access=%llu+%llu)",
577 577 (longlong_t)dn->dn_objset->
578 578 os_dsl_dataset->ds_object,
579 579 (longlong_t)dn->dn_object, dn->dn_datablksz,
580 580 (longlong_t)offset, (longlong_t)length);
581 581 rw_exit(&dn->dn_struct_rwlock);
582 582 return (SET_ERROR(EIO));
583 583 }
584 584 nblks = 1;
585 585 }
586 586 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
587 587
588 588 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
589 589 blkid = dbuf_whichblock(dn, 0, offset);
590 590 for (i = 0; i < nblks; i++) {
591 591 dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
592 592 if (db == NULL) {
593 593 rw_exit(&dn->dn_struct_rwlock);
594 594 dmu_buf_rele_array(dbp, nblks, tag);
595 595 zio_nowait(zio);
596 596 return (SET_ERROR(EIO));
597 597 }
598 598
599 599 /* initiate async i/o */
600 600 if (read)
601 601 (void) dbuf_read(db, zio, dbuf_flags);
602 602 dbp[i] = &db->db;
603 603 }
604 604
605 605 if ((flags & DMU_READ_NO_PREFETCH) == 0 &&
606 606 DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) {
607 607 dmu_zfetch(&dn->dn_zfetch, blkid, nblks,
608 608 read && DNODE_IS_CACHEABLE(dn));
609 609 }
610 610 rw_exit(&dn->dn_struct_rwlock);
611 611
612 612 /* wait for async i/o */
613 613 err = zio_wait(zio);
614 614 if (err) {
615 615 dmu_buf_rele_array(dbp, nblks, tag);
616 616 return (err);
617 617 }
618 618
619 619 /* wait for other io to complete */
620 620 if (read) {
621 621 for (i = 0; i < nblks; i++) {
622 622 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
623 623 mutex_enter(&db->db_mtx);
624 624 while (db->db_state == DB_READ ||
625 625 db->db_state == DB_FILL)
626 626 cv_wait(&db->db_changed, &db->db_mtx);
627 627 if (db->db_state == DB_UNCACHED)
628 628 err = SET_ERROR(EIO);
629 629 mutex_exit(&db->db_mtx);
630 630 if (err) {
631 631 dmu_buf_rele_array(dbp, nblks, tag);
632 632 return (err);
633 633 }
634 634 }
635 635 }
636 636
637 637 *numbufsp = nblks;
638 638 *dbpp = dbp;
639 639 return (0);
640 640 }
641 641
642 642 static int
643 643 dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
644 644 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
645 645 {
646 646 dnode_t *dn;
647 647 int err;
648 648
649 649 err = dnode_hold(os, object, FTAG, &dn);
650 650 if (err)
651 651 return (err);
652 652
653 653 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
654 654 numbufsp, dbpp, DMU_READ_PREFETCH);
655 655
656 656 dnode_rele(dn, FTAG);
657 657
658 658 return (err);
659 659 }
660 660
661 661 int
662 662 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
663 663 uint64_t length, boolean_t read, void *tag, int *numbufsp,
664 664 dmu_buf_t ***dbpp)
665 665 {
666 666 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
667 667 dnode_t *dn;
668 668 int err;
669 669
670 670 DB_DNODE_ENTER(db);
671 671 dn = DB_DNODE(db);
672 672 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
673 673 numbufsp, dbpp, DMU_READ_PREFETCH);
674 674 DB_DNODE_EXIT(db);
675 675
676 676 return (err);
677 677 }
678 678
679 679 void
680 680 dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
681 681 {
682 682 int i;
683 683 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
684 684
685 685 if (numbufs == 0)
686 686 return;
687 687
688 688 for (i = 0; i < numbufs; i++) {
689 689 if (dbp[i])
690 690 dbuf_rele(dbp[i], tag);
691 691 }
692 692
693 693 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
694 694 }
695 695
696 696 /*
697 697 * Issue prefetch i/os for the given blocks. If level is greater than 0, the
698 698 * indirect blocks prefeteched will be those that point to the blocks containing
699 699 * the data starting at offset, and continuing to offset + len.
700 700 *
701 701 * Note that if the indirect blocks above the blocks being prefetched are not
702 702 * in cache, they will be asychronously read in.
703 703 */
704 704 void
705 705 dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
706 706 uint64_t len, zio_priority_t pri)
707 707 {
708 708 dnode_t *dn;
709 709 uint64_t blkid;
710 710 int nblks, err;
711 711
712 712 if (len == 0) { /* they're interested in the bonus buffer */
713 713 dn = DMU_META_DNODE(os);
714 714
715 715 if (object == 0 || object >= DN_MAX_OBJECT)
716 716 return;
717 717
718 718 rw_enter(&dn->dn_struct_rwlock, RW_READER);
719 719 blkid = dbuf_whichblock(dn, level,
720 720 object * sizeof (dnode_phys_t));
721 721 dbuf_prefetch(dn, level, blkid, pri, 0);
722 722 rw_exit(&dn->dn_struct_rwlock);
723 723 return;
724 724 }
725 725
726 726 /*
727 727 * See comment before the definition of dmu_prefetch_max.
728 728 */
729 729 len = MIN(len, dmu_prefetch_max);
730 730
731 731 /*
732 732 * XXX - Note, if the dnode for the requested object is not
733 733 * already cached, we will do a *synchronous* read in the
734 734 * dnode_hold() call. The same is true for any indirects.
735 735 */
736 736 err = dnode_hold(os, object, FTAG, &dn);
737 737 if (err != 0)
738 738 return;
739 739
740 740 rw_enter(&dn->dn_struct_rwlock, RW_READER);
741 741 /*
742 742 * offset + len - 1 is the last byte we want to prefetch for, and offset
743 743 * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the
744 744 * last block we want to prefetch, and dbuf_whichblock(dn, level,
745 745 * offset) is the first. Then the number we need to prefetch is the
746 746 * last - first + 1.
747 747 */
748 748 if (level > 0 || dn->dn_datablkshift != 0) {
749 749 nblks = dbuf_whichblock(dn, level, offset + len - 1) -
750 750 dbuf_whichblock(dn, level, offset) + 1;
751 751 } else {
752 752 nblks = (offset < dn->dn_datablksz);
753 753 }
754 754
755 755 if (nblks != 0) {
756 756 blkid = dbuf_whichblock(dn, level, offset);
757 757 for (int i = 0; i < nblks; i++)
758 758 dbuf_prefetch(dn, level, blkid + i, pri, 0);
759 759 }
760 760
761 761 rw_exit(&dn->dn_struct_rwlock);
762 762
763 763 dnode_rele(dn, FTAG);
764 764 }
765 765
766 766 /*
767 767 * Get the next "chunk" of file data to free. We traverse the file from
768 768 * the end so that the file gets shorter over time (if we crashes in the
769 769 * middle, this will leave us in a better state). We find allocated file
770 770 * data by simply searching the allocated level 1 indirects.
771 771 *
772 772 * On input, *start should be the first offset that does not need to be
773 773 * freed (e.g. "offset + length"). On return, *start will be the first
774 774 * offset that should be freed.
775 775 */
776 776 static int
777 777 get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum)
778 778 {
779 779 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
780 780 /* bytes of data covered by a level-1 indirect block */
781 781 uint64_t iblkrange =
782 782 dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
783 783
784 784 ASSERT3U(minimum, <=, *start);
785 785
786 786 if (*start - minimum <= iblkrange * maxblks) {
787 787 *start = minimum;
788 788 return (0);
789 789 }
790 790 ASSERT(ISP2(iblkrange));
791 791
792 792 for (uint64_t blks = 0; *start > minimum && blks < maxblks; blks++) {
793 793 int err;
794 794
795 795 /*
796 796 * dnode_next_offset(BACKWARDS) will find an allocated L1
797 797 * indirect block at or before the input offset. We must
798 798 * decrement *start so that it is at the end of the region
799 799 * to search.
800 800 */
801 801 (*start)--;
802 802 err = dnode_next_offset(dn,
803 803 DNODE_FIND_BACKWARDS, start, 2, 1, 0);
804 804
805 805 /* if there are no indirect blocks before start, we are done */
806 806 if (err == ESRCH) {
807 807 *start = minimum;
808 808 break;
809 809 } else if (err != 0) {
810 810 return (err);
811 811 }
812 812
813 813 /* set start to the beginning of this L1 indirect */
814 814 *start = P2ALIGN(*start, iblkrange);
815 815 }
816 816 if (*start < minimum)
817 817 *start = minimum;
818 818 return (0);
819 819 }
820 820
821 821 /*
822 822 * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set,
823 823 * otherwise return false.
824 824 * Used below in dmu_free_long_range_impl() to enable abort when unmounting
825 825 */
826 826 /*ARGSUSED*/
827 827 static boolean_t
828 828 dmu_objset_zfs_unmounting(objset_t *os)
829 829 {
830 830 #ifdef _KERNEL
831 831 if (dmu_objset_type(os) == DMU_OST_ZFS)
832 832 return (zfs_get_vfs_flag_unmounted(os));
833 833 #endif
834 834 return (B_FALSE);
835 835 }
836 836
837 837 static int
838 838 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
839 839 uint64_t length)
840 840 {
841 841 uint64_t object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
842 842 int err;
843 843 uint64_t dirty_frees_threshold;
844 844 dsl_pool_t *dp = dmu_objset_pool(os);
845 845
846 846 if (offset >= object_size)
847 847 return (0);
848 848
849 849 if (zfs_per_txg_dirty_frees_percent <= 100)
850 850 dirty_frees_threshold =
851 851 zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100;
852 852 else
853 853 dirty_frees_threshold = zfs_dirty_data_max / 4;
854 854
855 855 if (length == DMU_OBJECT_END || offset + length > object_size)
856 856 length = object_size - offset;
857 857
858 858 while (length != 0) {
859 859 uint64_t chunk_end, chunk_begin, chunk_len;
860 860 uint64_t long_free_dirty_all_txgs = 0;
861 861 dmu_tx_t *tx;
862 862
863 863 if (dmu_objset_zfs_unmounting(dn->dn_objset))
864 864 return (SET_ERROR(EINTR));
865 865
866 866 chunk_end = chunk_begin = offset + length;
867 867
868 868 /* move chunk_begin backwards to the beginning of this chunk */
869 869 err = get_next_chunk(dn, &chunk_begin, offset);
870 870 if (err)
871 871 return (err);
872 872 ASSERT3U(chunk_begin, >=, offset);
873 873 ASSERT3U(chunk_begin, <=, chunk_end);
874 874
875 875 chunk_len = chunk_end - chunk_begin;
876 876
877 877 mutex_enter(&dp->dp_lock);
878 878 for (int t = 0; t < TXG_SIZE; t++) {
879 879 long_free_dirty_all_txgs +=
880 880 dp->dp_long_free_dirty_pertxg[t];
881 881 }
882 882 mutex_exit(&dp->dp_lock);
883 883
884 884 /*
885 885 * To avoid filling up a TXG with just frees wait for
886 886 * the next TXG to open before freeing more chunks if
887 887 * we have reached the threshold of frees
888 888 */
889 889 if (dirty_frees_threshold != 0 &&
890 890 long_free_dirty_all_txgs >= dirty_frees_threshold) {
891 891 txg_wait_open(dp, 0);
892 892 continue;
893 893 }
894 894
895 895 tx = dmu_tx_create(os);
896 896 dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len);
897 897
898 898 /*
899 899 * Mark this transaction as typically resulting in a net
900 900 * reduction in space used.
901 901 */
902 902 dmu_tx_mark_netfree(tx);
903 903 err = dmu_tx_assign(tx, TXG_WAIT);
904 904 if (err) {
905 905 dmu_tx_abort(tx);
906 906 return (err);
907 907 }
908 908
909 909 mutex_enter(&dp->dp_lock);
910 910 dp->dp_long_free_dirty_pertxg[dmu_tx_get_txg(tx) & TXG_MASK] +=
911 911 chunk_len;
912 912 mutex_exit(&dp->dp_lock);
913 913 DTRACE_PROBE3(free__long__range,
914 914 uint64_t, long_free_dirty_all_txgs, uint64_t, chunk_len,
915 915 uint64_t, dmu_tx_get_txg(tx));
916 916 dnode_free_range(dn, chunk_begin, chunk_len, tx);
917 917
918 918 dmu_tx_commit(tx);
919 919
920 920 length -= chunk_len;
921 921 }
922 922 return (0);
923 923 }
924 924
925 925 int
926 926 dmu_free_long_range(objset_t *os, uint64_t object,
927 927 uint64_t offset, uint64_t length)
928 928 {
929 929 dnode_t *dn;
930 930 int err;
931 931
932 932 err = dnode_hold(os, object, FTAG, &dn);
933 933 if (err != 0)
934 934 return (err);
935 935 err = dmu_free_long_range_impl(os, dn, offset, length);
936 936
937 937 /*
938 938 * It is important to zero out the maxblkid when freeing the entire
939 939 * file, so that (a) subsequent calls to dmu_free_long_range_impl()
940 940 * will take the fast path, and (b) dnode_reallocate() can verify
941 941 * that the entire file has been freed.
942 942 */
943 943 if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
944 944 dn->dn_maxblkid = 0;
945 945
946 946 dnode_rele(dn, FTAG);
947 947 return (err);
948 948 }
949 949
950 950 int
951 951 dmu_free_long_object(objset_t *os, uint64_t object)
952 952 {
953 953 dmu_tx_t *tx;
954 954 int err;
955 955
956 956 err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
957 957 if (err != 0)
958 958 return (err);
959 959
960 960 tx = dmu_tx_create(os);
961 961 dmu_tx_hold_bonus(tx, object);
962 962 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
963 963 dmu_tx_mark_netfree(tx);
964 964 err = dmu_tx_assign(tx, TXG_WAIT);
965 965 if (err == 0) {
966 966 if (err == 0)
967 967 err = dmu_object_free(os, object, tx);
968 968
969 969 dmu_tx_commit(tx);
970 970 } else {
971 971 dmu_tx_abort(tx);
972 972 }
973 973
974 974 return (err);
975 975 }
976 976
977 977 int
978 978 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
979 979 uint64_t size, dmu_tx_t *tx)
980 980 {
981 981 dnode_t *dn;
982 982 int err = dnode_hold(os, object, FTAG, &dn);
983 983 if (err)
984 984 return (err);
985 985 ASSERT(offset < UINT64_MAX);
986 986 ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset);
987 987 dnode_free_range(dn, offset, size, tx);
988 988 dnode_rele(dn, FTAG);
989 989 return (0);
990 990 }
991 991
992 992 static int
993 993 dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size,
994 994 void *buf, uint32_t flags)
995 995 {
996 996 dmu_buf_t **dbp;
997 997 int numbufs, err = 0;
998 998
999 999 /*
1000 1000 * Deal with odd block sizes, where there can't be data past the first
1001 1001 * block. If we ever do the tail block optimization, we will need to
1002 1002 * handle that here as well.
1003 1003 */
1004 1004 if (dn->dn_maxblkid == 0) {
1005 1005 int newsz = offset > dn->dn_datablksz ? 0 :
1006 1006 MIN(size, dn->dn_datablksz - offset);
1007 1007 bzero((char *)buf + newsz, size - newsz);
1008 1008 size = newsz;
1009 1009 }
1010 1010
1011 1011 while (size > 0) {
1012 1012 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
1013 1013 int i;
1014 1014
1015 1015 /*
1016 1016 * NB: we could do this block-at-a-time, but it's nice
1017 1017 * to be reading in parallel.
1018 1018 */
1019 1019 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
1020 1020 TRUE, FTAG, &numbufs, &dbp, flags);
1021 1021 if (err)
1022 1022 break;
1023 1023
1024 1024 for (i = 0; i < numbufs; i++) {
1025 1025 int tocpy;
1026 1026 int bufoff;
1027 1027 dmu_buf_t *db = dbp[i];
1028 1028
1029 1029 ASSERT(size > 0);
1030 1030
1031 1031 bufoff = offset - db->db_offset;
1032 1032 tocpy = (int)MIN(db->db_size - bufoff, size);
1033 1033
1034 1034 bcopy((char *)db->db_data + bufoff, buf, tocpy);
1035 1035
1036 1036 offset += tocpy;
1037 1037 size -= tocpy;
1038 1038 buf = (char *)buf + tocpy;
1039 1039 }
1040 1040 dmu_buf_rele_array(dbp, numbufs, FTAG);
1041 1041 }
1042 1042 return (err);
1043 1043 }
1044 1044
1045 1045 int
1046 1046 dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1047 1047 void *buf, uint32_t flags)
1048 1048 {
1049 1049 dnode_t *dn;
1050 1050 int err;
1051 1051
1052 1052 err = dnode_hold(os, object, FTAG, &dn);
1053 1053 if (err != 0)
1054 1054 return (err);
1055 1055
1056 1056 err = dmu_read_impl(dn, offset, size, buf, flags);
1057 1057 dnode_rele(dn, FTAG);
1058 1058 return (err);
1059 1059 }
1060 1060
1061 1061 int
1062 1062 dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf,
1063 1063 uint32_t flags)
1064 1064 {
1065 1065 return (dmu_read_impl(dn, offset, size, buf, flags));
1066 1066 }
1067 1067
1068 1068 static void
1069 1069 dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size,
1070 1070 const void *buf, dmu_tx_t *tx)
1071 1071 {
1072 1072 int i;
1073 1073
1074 1074 for (i = 0; i < numbufs; i++) {
1075 1075 int tocpy;
1076 1076 int bufoff;
1077 1077 dmu_buf_t *db = dbp[i];
1078 1078
1079 1079 ASSERT(size > 0);
1080 1080
1081 1081 bufoff = offset - db->db_offset;
1082 1082 tocpy = (int)MIN(db->db_size - bufoff, size);
1083 1083
1084 1084 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1085 1085
1086 1086 if (tocpy == db->db_size)
1087 1087 dmu_buf_will_fill(db, tx);
1088 1088 else
1089 1089 dmu_buf_will_dirty(db, tx);
1090 1090
1091 1091 bcopy(buf, (char *)db->db_data + bufoff, tocpy);
1092 1092
1093 1093 if (tocpy == db->db_size)
1094 1094 dmu_buf_fill_done(db, tx);
1095 1095
1096 1096 offset += tocpy;
1097 1097 size -= tocpy;
1098 1098 buf = (char *)buf + tocpy;
1099 1099 }
1100 1100 }
1101 1101
1102 1102 void
1103 1103 dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1104 1104 const void *buf, dmu_tx_t *tx)
1105 1105 {
1106 1106 dmu_buf_t **dbp;
1107 1107 int numbufs;
1108 1108
1109 1109 if (size == 0)
1110 1110 return;
1111 1111
1112 1112 VERIFY0(dmu_buf_hold_array(os, object, offset, size,
1113 1113 FALSE, FTAG, &numbufs, &dbp));
1114 1114 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1115 1115 dmu_buf_rele_array(dbp, numbufs, FTAG);
1116 1116 }
1117 1117
1118 1118 void
1119 1119 dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
1120 1120 const void *buf, dmu_tx_t *tx)
1121 1121 {
1122 1122 dmu_buf_t **dbp;
1123 1123 int numbufs;
1124 1124
1125 1125 if (size == 0)
1126 1126 return;
1127 1127
1128 1128 VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size,
1129 1129 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH));
1130 1130 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1131 1131 dmu_buf_rele_array(dbp, numbufs, FTAG);
1132 1132 }
1133 1133
1134 1134 static int
1135 1135 dmu_object_remap_one_indirect(objset_t *os, dnode_t *dn,
1136 1136 uint64_t last_removal_txg, uint64_t offset)
1137 1137 {
1138 1138 uint64_t l1blkid = dbuf_whichblock(dn, 1, offset);
1139 1139 int err = 0;
1140 1140
1141 1141 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1142 1142 dmu_buf_impl_t *dbuf = dbuf_hold_level(dn, 1, l1blkid, FTAG);
1143 1143 ASSERT3P(dbuf, !=, NULL);
1144 1144
1145 1145 /*
1146 1146 * If the block hasn't been written yet, this default will ensure
1147 1147 * we don't try to remap it.
1148 1148 */
1149 1149 uint64_t birth = UINT64_MAX;
1150 1150 ASSERT3U(last_removal_txg, !=, UINT64_MAX);
1151 1151 if (dbuf->db_blkptr != NULL)
1152 1152 birth = dbuf->db_blkptr->blk_birth;
1153 1153 rw_exit(&dn->dn_struct_rwlock);
1154 1154
1155 1155 /*
1156 1156 * If this L1 was already written after the last removal, then we've
1157 1157 * already tried to remap it.
1158 1158 */
1159 1159 if (birth <= last_removal_txg &&
1160 1160 dbuf_read(dbuf, NULL, DB_RF_MUST_SUCCEED) == 0 &&
1161 1161 dbuf_can_remap(dbuf)) {
1162 1162 dmu_tx_t *tx = dmu_tx_create(os);
1163 1163 dmu_tx_hold_remap_l1indirect(tx, dn->dn_object);
1164 1164 err = dmu_tx_assign(tx, TXG_WAIT);
1165 1165 if (err == 0) {
1166 1166 (void) dbuf_dirty(dbuf, tx);
1167 1167 dmu_tx_commit(tx);
1168 1168 } else {
1169 1169 dmu_tx_abort(tx);
1170 1170 }
1171 1171 }
1172 1172
1173 1173 dbuf_rele(dbuf, FTAG);
1174 1174
1175 1175 delay(zfs_object_remap_one_indirect_delay_ticks);
1176 1176
1177 1177 return (err);
1178 1178 }
1179 1179
1180 1180 /*
1181 1181 * Remap all blockpointers in the object, if possible, so that they reference
1182 1182 * only concrete vdevs.
1183 1183 *
1184 1184 * To do this, iterate over the L0 blockpointers and remap any that reference
1185 1185 * an indirect vdev. Note that we only examine L0 blockpointers; since we
1186 1186 * cannot guarantee that we can remap all blockpointer anyways (due to split
1187 1187 * blocks), we do not want to make the code unnecessarily complicated to
1188 1188 * catch the unlikely case that there is an L1 block on an indirect vdev that
1189 1189 * contains no indirect blockpointers.
1190 1190 */
1191 1191 int
1192 1192 dmu_object_remap_indirects(objset_t *os, uint64_t object,
1193 1193 uint64_t last_removal_txg)
1194 1194 {
1195 1195 uint64_t offset, l1span;
1196 1196 int err;
1197 1197 dnode_t *dn;
1198 1198
1199 1199 err = dnode_hold(os, object, FTAG, &dn);
1200 1200 if (err != 0) {
1201 1201 return (err);
1202 1202 }
1203 1203
1204 1204 if (dn->dn_nlevels <= 1) {
1205 1205 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1206 1206 err = SET_ERROR(EINTR);
1207 1207 }
1208 1208
1209 1209 /*
1210 1210 * If the dnode has no indirect blocks, we cannot dirty them.
1211 1211 * We still want to remap the blkptr(s) in the dnode if
1212 1212 * appropriate, so mark it as dirty.
1213 1213 */
1214 1214 if (err == 0 && dnode_needs_remap(dn)) {
1215 1215 dmu_tx_t *tx = dmu_tx_create(os);
1216 1216 dmu_tx_hold_bonus(tx, dn->dn_object);
1217 1217 if ((err = dmu_tx_assign(tx, TXG_WAIT)) == 0) {
1218 1218 dnode_setdirty(dn, tx);
1219 1219 dmu_tx_commit(tx);
1220 1220 } else {
1221 1221 dmu_tx_abort(tx);
1222 1222 }
1223 1223 }
1224 1224
1225 1225 dnode_rele(dn, FTAG);
1226 1226 return (err);
1227 1227 }
1228 1228
1229 1229 offset = 0;
1230 1230 l1span = 1ULL << (dn->dn_indblkshift - SPA_BLKPTRSHIFT +
1231 1231 dn->dn_datablkshift);
1232 1232 /*
1233 1233 * Find the next L1 indirect that is not a hole.
1234 1234 */
1235 1235 while (dnode_next_offset(dn, 0, &offset, 2, 1, 0) == 0) {
1236 1236 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1237 1237 err = SET_ERROR(EINTR);
1238 1238 break;
1239 1239 }
1240 1240 if ((err = dmu_object_remap_one_indirect(os, dn,
1241 1241 last_removal_txg, offset)) != 0) {
1242 1242 break;
1243 1243 }
1244 1244 offset += l1span;
1245 1245 }
1246 1246
1247 1247 dnode_rele(dn, FTAG);
1248 1248 return (err);
1249 1249 }
1250 1250
1251 1251 void
1252 1252 dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1253 1253 dmu_tx_t *tx)
1254 1254 {
1255 1255 dmu_buf_t **dbp;
1256 1256 int numbufs, i;
1257 1257
1258 1258 if (size == 0)
1259 1259 return;
1260 1260
1261 1261 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
1262 1262 FALSE, FTAG, &numbufs, &dbp));
1263 1263
1264 1264 for (i = 0; i < numbufs; i++) {
1265 1265 dmu_buf_t *db = dbp[i];
1266 1266
1267 1267 dmu_buf_will_not_fill(db, tx);
1268 1268 }
1269 1269 dmu_buf_rele_array(dbp, numbufs, FTAG);
1270 1270 }
1271 1271
1272 1272 void
1273 1273 dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
1274 1274 void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
1275 1275 int compressed_size, int byteorder, dmu_tx_t *tx)
1276 1276 {
1277 1277 dmu_buf_t *db;
1278 1278
1279 1279 ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES);
1280 1280 ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS);
1281 1281 VERIFY0(dmu_buf_hold_noread(os, object, offset,
1282 1282 FTAG, &db));
1283 1283
1284 1284 dmu_buf_write_embedded(db,
1285 1285 data, (bp_embedded_type_t)etype, (enum zio_compress)comp,
1286 1286 uncompressed_size, compressed_size, byteorder, tx);
1287 1287
1288 1288 dmu_buf_rele(db, FTAG);
1289 1289 }
1290 1290
1291 1291 /*
1292 1292 * DMU support for xuio
1293 1293 */
1294 1294 kstat_t *xuio_ksp = NULL;
1295 1295
1296 1296 int
1297 1297 dmu_xuio_init(xuio_t *xuio, int nblk)
1298 1298 {
1299 1299 dmu_xuio_t *priv;
1300 1300 uio_t *uio = &xuio->xu_uio;
1301 1301
1302 1302 uio->uio_iovcnt = nblk;
1303 1303 uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
1304 1304
1305 1305 priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
1306 1306 priv->cnt = nblk;
1307 1307 priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
1308 1308 priv->iovp = uio->uio_iov;
1309 1309 XUIO_XUZC_PRIV(xuio) = priv;
1310 1310
1311 1311 if (XUIO_XUZC_RW(xuio) == UIO_READ)
1312 1312 XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk);
1313 1313 else
1314 1314 XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk);
1315 1315
1316 1316 return (0);
1317 1317 }
1318 1318
1319 1319 void
1320 1320 dmu_xuio_fini(xuio_t *xuio)
1321 1321 {
1322 1322 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1323 1323 int nblk = priv->cnt;
1324 1324
1325 1325 kmem_free(priv->iovp, nblk * sizeof (iovec_t));
1326 1326 kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *));
1327 1327 kmem_free(priv, sizeof (dmu_xuio_t));
1328 1328
1329 1329 if (XUIO_XUZC_RW(xuio) == UIO_READ)
1330 1330 XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk);
1331 1331 else
1332 1332 XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk);
1333 1333 }
1334 1334
1335 1335 /*
1336 1336 * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf }
1337 1337 * and increase priv->next by 1.
1338 1338 */
1339 1339 int
1340 1340 dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n)
1341 1341 {
1342 1342 struct iovec *iov;
1343 1343 uio_t *uio = &xuio->xu_uio;
1344 1344 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1345 1345 int i = priv->next++;
1346 1346
1347 1347 ASSERT(i < priv->cnt);
1348 1348 ASSERT(off + n <= arc_buf_lsize(abuf));
1349 1349 iov = uio->uio_iov + i;
1350 1350 iov->iov_base = (char *)abuf->b_data + off;
1351 1351 iov->iov_len = n;
1352 1352 priv->bufs[i] = abuf;
1353 1353 return (0);
1354 1354 }
1355 1355
1356 1356 int
1357 1357 dmu_xuio_cnt(xuio_t *xuio)
1358 1358 {
1359 1359 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1360 1360 return (priv->cnt);
1361 1361 }
1362 1362
1363 1363 arc_buf_t *
1364 1364 dmu_xuio_arcbuf(xuio_t *xuio, int i)
1365 1365 {
1366 1366 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1367 1367
1368 1368 ASSERT(i < priv->cnt);
1369 1369 return (priv->bufs[i]);
1370 1370 }
1371 1371
1372 1372 void
1373 1373 dmu_xuio_clear(xuio_t *xuio, int i)
1374 1374 {
1375 1375 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1376 1376
1377 1377 ASSERT(i < priv->cnt);
1378 1378 priv->bufs[i] = NULL;
1379 1379 }
1380 1380
1381 1381 static void
1382 1382 xuio_stat_init(void)
1383 1383 {
1384 1384 xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc",
1385 1385 KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t),
1386 1386 KSTAT_FLAG_VIRTUAL);
1387 1387 if (xuio_ksp != NULL) {
1388 1388 xuio_ksp->ks_data = &xuio_stats;
1389 1389 kstat_install(xuio_ksp);
1390 1390 }
1391 1391 }
1392 1392
1393 1393 static void
1394 1394 xuio_stat_fini(void)
1395 1395 {
1396 1396 if (xuio_ksp != NULL) {
1397 1397 kstat_delete(xuio_ksp);
1398 1398 xuio_ksp = NULL;
1399 1399 }
1400 1400 }
1401 1401
1402 1402 void
1403 1403 xuio_stat_wbuf_copied(void)
1404 1404 {
1405 1405 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1406 1406 }
1407 1407
1408 1408 void
1409 1409 xuio_stat_wbuf_nocopy(void)
1410 1410 {
1411 1411 XUIOSTAT_BUMP(xuiostat_wbuf_nocopy);
1412 1412 }
1413 1413
1414 1414 #ifdef _KERNEL
1415 1415 int
1416 1416 dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size)
1417 1417 {
1418 1418 dmu_buf_t **dbp;
1419 1419 int numbufs, i, err;
1420 1420 xuio_t *xuio = NULL;
1421 1421
1422 1422 /*
1423 1423 * NB: we could do this block-at-a-time, but it's nice
1424 1424 * to be reading in parallel.
1425 1425 */
1426 1426 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1427 1427 TRUE, FTAG, &numbufs, &dbp, 0);
1428 1428 if (err)
1429 1429 return (err);
1430 1430
1431 1431 if (uio->uio_extflg == UIO_XUIO)
1432 1432 xuio = (xuio_t *)uio;
1433 1433
1434 1434 for (i = 0; i < numbufs; i++) {
1435 1435 int tocpy;
1436 1436 int bufoff;
1437 1437 dmu_buf_t *db = dbp[i];
1438 1438
1439 1439 ASSERT(size > 0);
1440 1440
1441 1441 bufoff = uio->uio_loffset - db->db_offset;
1442 1442 tocpy = (int)MIN(db->db_size - bufoff, size);
1443 1443
1444 1444 if (xuio) {
1445 1445 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
1446 1446 arc_buf_t *dbuf_abuf = dbi->db_buf;
1447 1447 arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
1448 1448 err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
1449 1449 if (!err) {
1450 1450 uio->uio_resid -= tocpy;
1451 1451 uio->uio_loffset += tocpy;
1452 1452 }
1453 1453
1454 1454 if (abuf == dbuf_abuf)
1455 1455 XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
1456 1456 else
1457 1457 XUIOSTAT_BUMP(xuiostat_rbuf_copied);
1458 1458 } else {
1459 1459 err = uiomove((char *)db->db_data + bufoff, tocpy,
1460 1460 UIO_READ, uio);
1461 1461 }
1462 1462 if (err)
1463 1463 break;
1464 1464
1465 1465 size -= tocpy;
1466 1466 }
1467 1467 dmu_buf_rele_array(dbp, numbufs, FTAG);
1468 1468
1469 1469 return (err);
1470 1470 }
1471 1471
1472 1472 /*
1473 1473 * Read 'size' bytes into the uio buffer.
1474 1474 * From object zdb->db_object.
1475 1475 * Starting at offset uio->uio_loffset.
1476 1476 *
1477 1477 * If the caller already has a dbuf in the target object
1478 1478 * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
1479 1479 * because we don't have to find the dnode_t for the object.
1480 1480 */
1481 1481 int
1482 1482 dmu_read_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size)
1483 1483 {
1484 1484 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1485 1485 dnode_t *dn;
1486 1486 int err;
1487 1487
1488 1488 if (size == 0)
1489 1489 return (0);
1490 1490
1491 1491 DB_DNODE_ENTER(db);
1492 1492 dn = DB_DNODE(db);
1493 1493 err = dmu_read_uio_dnode(dn, uio, size);
1494 1494 DB_DNODE_EXIT(db);
1495 1495
1496 1496 return (err);
1497 1497 }
1498 1498
1499 1499 /*
1500 1500 * Read 'size' bytes into the uio buffer.
1501 1501 * From the specified object
1502 1502 * Starting at offset uio->uio_loffset.
1503 1503 */
1504 1504 int
1505 1505 dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
1506 1506 {
1507 1507 dnode_t *dn;
1508 1508 int err;
1509 1509
1510 1510 if (size == 0)
1511 1511 return (0);
1512 1512
1513 1513 err = dnode_hold(os, object, FTAG, &dn);
1514 1514 if (err)
1515 1515 return (err);
1516 1516
1517 1517 err = dmu_read_uio_dnode(dn, uio, size);
1518 1518
1519 1519 dnode_rele(dn, FTAG);
1520 1520
1521 1521 return (err);
1522 1522 }
1523 1523
1524 1524 int
1525 1525 dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
1526 1526 {
1527 1527 dmu_buf_t **dbp;
1528 1528 int numbufs;
1529 1529 int err = 0;
1530 1530 int i;
1531 1531
1532 1532 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1533 1533 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1534 1534 if (err)
1535 1535 return (err);
1536 1536
1537 1537 for (i = 0; i < numbufs; i++) {
1538 1538 int tocpy;
1539 1539 int bufoff;
1540 1540 dmu_buf_t *db = dbp[i];
1541 1541
1542 1542 ASSERT(size > 0);
1543 1543
1544 1544 bufoff = uio->uio_loffset - db->db_offset;
1545 1545 tocpy = (int)MIN(db->db_size - bufoff, size);
1546 1546
1547 1547 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1548 1548
1549 1549 if (tocpy == db->db_size)
1550 1550 dmu_buf_will_fill(db, tx);
1551 1551 else
1552 1552 dmu_buf_will_dirty(db, tx);
1553 1553
1554 1554 /*
1555 1555 * XXX uiomove could block forever (eg. nfs-backed
1556 1556 * pages). There needs to be a uiolockdown() function
1557 1557 * to lock the pages in memory, so that uiomove won't
1558 1558 * block.
1559 1559 */
1560 1560 err = uiomove((char *)db->db_data + bufoff, tocpy,
1561 1561 UIO_WRITE, uio);
1562 1562
1563 1563 if (tocpy == db->db_size)
1564 1564 dmu_buf_fill_done(db, tx);
1565 1565
1566 1566 if (err)
1567 1567 break;
1568 1568
1569 1569 size -= tocpy;
1570 1570 }
1571 1571
1572 1572 dmu_buf_rele_array(dbp, numbufs, FTAG);
1573 1573 return (err);
1574 1574 }
1575 1575
1576 1576 /*
1577 1577 * Write 'size' bytes from the uio buffer.
1578 1578 * To object zdb->db_object.
1579 1579 * Starting at offset uio->uio_loffset.
1580 1580 *
1581 1581 * If the caller already has a dbuf in the target object
1582 1582 * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
1583 1583 * because we don't have to find the dnode_t for the object.
1584 1584 */
1585 1585 int
1586 1586 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
1587 1587 dmu_tx_t *tx)
1588 1588 {
1589 1589 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1590 1590 dnode_t *dn;
1591 1591 int err;
1592 1592
1593 1593 if (size == 0)
1594 1594 return (0);
1595 1595
1596 1596 DB_DNODE_ENTER(db);
1597 1597 dn = DB_DNODE(db);
1598 1598 err = dmu_write_uio_dnode(dn, uio, size, tx);
1599 1599 DB_DNODE_EXIT(db);
1600 1600
1601 1601 return (err);
1602 1602 }
1603 1603
1604 1604 /*
1605 1605 * Write 'size' bytes from the uio buffer.
1606 1606 * To the specified object.
1607 1607 * Starting at offset uio->uio_loffset.
1608 1608 */
1609 1609 int
1610 1610 dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
1611 1611 dmu_tx_t *tx)
1612 1612 {
1613 1613 dnode_t *dn;
1614 1614 int err;
1615 1615
1616 1616 if (size == 0)
1617 1617 return (0);
1618 1618
1619 1619 err = dnode_hold(os, object, FTAG, &dn);
1620 1620 if (err)
1621 1621 return (err);
1622 1622
1623 1623 err = dmu_write_uio_dnode(dn, uio, size, tx);
1624 1624
1625 1625 dnode_rele(dn, FTAG);
1626 1626
1627 1627 return (err);
1628 1628 }
1629 1629
1630 1630 int
1631 1631 dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1632 1632 page_t *pp, dmu_tx_t *tx)
1633 1633 {
1634 1634 dmu_buf_t **dbp;
1635 1635 int numbufs, i;
1636 1636 int err;
1637 1637
1638 1638 if (size == 0)
1639 1639 return (0);
1640 1640
1641 1641 err = dmu_buf_hold_array(os, object, offset, size,
1642 1642 FALSE, FTAG, &numbufs, &dbp);
1643 1643 if (err)
1644 1644 return (err);
1645 1645
1646 1646 for (i = 0; i < numbufs; i++) {
1647 1647 int tocpy, copied, thiscpy;
1648 1648 int bufoff;
1649 1649 dmu_buf_t *db = dbp[i];
1650 1650 caddr_t va;
1651 1651
1652 1652 ASSERT(size > 0);
1653 1653 ASSERT3U(db->db_size, >=, PAGESIZE);
1654 1654
1655 1655 bufoff = offset - db->db_offset;
1656 1656 tocpy = (int)MIN(db->db_size - bufoff, size);
1657 1657
1658 1658 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1659 1659
1660 1660 if (tocpy == db->db_size)
1661 1661 dmu_buf_will_fill(db, tx);
1662 1662 else
1663 1663 dmu_buf_will_dirty(db, tx);
1664 1664
1665 1665 for (copied = 0; copied < tocpy; copied += PAGESIZE) {
1666 1666 ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff);
1667 1667 thiscpy = MIN(PAGESIZE, tocpy - copied);
1668 1668 va = zfs_map_page(pp, S_READ);
1669 1669 bcopy(va, (char *)db->db_data + bufoff, thiscpy);
1670 1670 zfs_unmap_page(pp, va);
1671 1671 pp = pp->p_next;
1672 1672 bufoff += PAGESIZE;
1673 1673 }
1674 1674
1675 1675 if (tocpy == db->db_size)
1676 1676 dmu_buf_fill_done(db, tx);
1677 1677
1678 1678 offset += tocpy;
1679 1679 size -= tocpy;
1680 1680 }
1681 1681 dmu_buf_rele_array(dbp, numbufs, FTAG);
1682 1682 return (err);
1683 1683 }
1684 1684 #endif
1685 1685
1686 1686 /*
1687 1687 * Allocate a loaned anonymous arc buffer.
1688 1688 */
1689 1689 arc_buf_t *
1690 1690 dmu_request_arcbuf(dmu_buf_t *handle, int size)
1691 1691 {
1692 1692 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
1693 1693
1694 1694 return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size));
1695 1695 }
1696 1696
1697 1697 /*
1698 1698 * Free a loaned arc buffer.
1699 1699 */
1700 1700 void
1701 1701 dmu_return_arcbuf(arc_buf_t *buf)
1702 1702 {
1703 1703 arc_return_buf(buf, FTAG);
1704 1704 arc_buf_destroy(buf, FTAG);
1705 1705 }
1706 1706
1707 1707 void
1708 1708 dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
1709 1709 dmu_buf_t *handle, dmu_tx_t *tx)
1710 1710 {
1711 1711 dmu_buf_t *dst_handle;
1712 1712 dmu_buf_impl_t *dstdb;
1713 1713 dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle;
1714 1714 dmu_object_type_t type;
1715 1715 arc_buf_t *abuf;
1716 1716 uint64_t datalen;
1717 1717 boolean_t byteorder;
1718 1718 uint8_t salt[ZIO_DATA_SALT_LEN];
1719 1719 uint8_t iv[ZIO_DATA_IV_LEN];
1720 1720 uint8_t mac[ZIO_DATA_MAC_LEN];
1721 1721
1722 1722 ASSERT3P(srcdb->db_buf, !=, NULL);
1723 1723
1724 1724 /* hold the db that we want to write to */
1725 1725 VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle,
1726 1726 DMU_READ_NO_DECRYPT));
1727 1727 dstdb = (dmu_buf_impl_t *)dst_handle;
1728 1728 datalen = arc_buf_size(srcdb->db_buf);
1729 1729
1730 1730 DB_DNODE_ENTER(dstdb);
1731 1731 type = DB_DNODE(dstdb)->dn_type;
1732 1732 DB_DNODE_EXIT(dstdb);
1733 1733
1734 1734 /* allocated an arc buffer that matches the type of srcdb->db_buf */
1735 1735 if (arc_is_encrypted(srcdb->db_buf)) {
1736 1736 arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac);
1737 1737 abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os),
1738 1738 byteorder, salt, iv, mac, type,
1739 1739 datalen, arc_buf_lsize(srcdb->db_buf),
1740 1740 arc_get_compression(srcdb->db_buf));
1741 1741 } else {
1742 1742 /* we won't get a compressed db back from dmu_buf_hold() */
1743 1743 ASSERT3U(arc_get_compression(srcdb->db_buf),
1744 1744 ==, ZIO_COMPRESS_OFF);
1745 1745 abuf = arc_loan_buf(os->os_spa,
1746 1746 DMU_OT_IS_METADATA(type), datalen);
1747 1747 }
1748 1748
1749 1749 ASSERT3U(datalen, ==, arc_buf_size(abuf));
1750 1750
1751 1751 /* copy the data to the new buffer and assign it to the dstdb */
1752 1752 bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen);
1753 1753 dbuf_assign_arcbuf(dstdb, abuf, tx);
1754 1754 dmu_buf_rele(dst_handle, FTAG);
1755 1755 }
1756 1756
1757 1757 /*
1758 1758 * When possible directly assign passed loaned arc buffer to a dbuf.
1759 1759 * If this is not possible copy the contents of passed arc buf via
1760 1760 * dmu_write().
1761 1761 */
1762 1762 int
1763 1763 dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
1764 1764 dmu_tx_t *tx)
1765 1765 {
1766 1766 dmu_buf_impl_t *db;
1767 1767 objset_t *os = dn->dn_objset;
1768 1768 uint64_t object = dn->dn_object;
1769 1769 uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
1770 1770 uint64_t blkid;
1771 1771
1772 1772 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1773 1773 blkid = dbuf_whichblock(dn, 0, offset);
1774 1774 db = dbuf_hold(dn, blkid, FTAG);
1775 1775 if (db == NULL)
1776 1776 return (SET_ERROR(EIO));
1777 1777 rw_exit(&dn->dn_struct_rwlock);
1778 1778
1779 1779 /*
1780 1780 * We can only assign if the offset is aligned, the arc buf is the
1781 1781 * same size as the dbuf, and the dbuf is not metadata.
1782 1782 */
1783 1783 if (offset == db->db.db_offset && blksz == db->db.db_size) {
1784 1784 dbuf_assign_arcbuf(db, buf, tx);
1785 1785 dbuf_rele(db, FTAG);
1786 1786 } else {
1787 1787 /* compressed bufs must always be assignable to their dbuf */
1788 1788 ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
1789 1789 ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
1790 1790
1791 1791 os = dn->dn_objset;
1792 1792 object = dn->dn_object;
1793 1793 dbuf_rele(db, FTAG);
1794 1794 dmu_write(os, object, offset, blksz, buf->b_data, tx);
1795 1795 dmu_return_arcbuf(buf);
1796 1796 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1797 1797 }
1798 1798
1799 1799 return (0);
1800 1800 }
1801 1801
1802 1802 int
1803 1803 dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
1804 1804 dmu_tx_t *tx)
1805 1805 {
1806 1806 int err;
1807 1807 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1808 1808
1809 1809 DB_DNODE_ENTER(dbuf);
1810 1810 err = dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx);
1811 1811 DB_DNODE_EXIT(dbuf);
1812 1812
1813 1813 return (err);
1814 1814 }
1815 1815
1816 1816 typedef struct {
1817 1817 dbuf_dirty_record_t *dsa_dr;
1818 1818 dmu_sync_cb_t *dsa_done;
1819 1819 zgd_t *dsa_zgd;
1820 1820 dmu_tx_t *dsa_tx;
1821 1821 } dmu_sync_arg_t;
1822 1822
1823 1823 /* ARGSUSED */
1824 1824 static void
1825 1825 dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1826 1826 {
1827 1827 dmu_sync_arg_t *dsa = varg;
1828 1828 dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1829 1829 blkptr_t *bp = zio->io_bp;
1830 1830
1831 1831 if (zio->io_error == 0) {
1832 1832 if (BP_IS_HOLE(bp)) {
1833 1833 /*
1834 1834 * A block of zeros may compress to a hole, but the
1835 1835 * block size still needs to be known for replay.
1836 1836 */
1837 1837 BP_SET_LSIZE(bp, db->db_size);
1838 1838 } else if (!BP_IS_EMBEDDED(bp)) {
1839 1839 ASSERT(BP_GET_LEVEL(bp) == 0);
1840 1840 BP_SET_FILL(bp, 1);
1841 1841 }
1842 1842 }
1843 1843 }
1844 1844
1845 1845 static void
1846 1846 dmu_sync_late_arrival_ready(zio_t *zio)
1847 1847 {
1848 1848 dmu_sync_ready(zio, NULL, zio->io_private);
1849 1849 }
1850 1850
1851 1851 /* ARGSUSED */
1852 1852 static void
1853 1853 dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1854 1854 {
1855 1855 dmu_sync_arg_t *dsa = varg;
1856 1856 dbuf_dirty_record_t *dr = dsa->dsa_dr;
1857 1857 dmu_buf_impl_t *db = dr->dr_dbuf;
1858 1858 zgd_t *zgd = dsa->dsa_zgd;
1859 1859
1860 1860 /*
1861 1861 * Record the vdev(s) backing this blkptr so they can be flushed after
1862 1862 * the writes for the lwb have completed.
1863 1863 */
1864 1864 if (zio->io_error == 0) {
1865 1865 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1866 1866 }
1867 1867
1868 1868 mutex_enter(&db->db_mtx);
1869 1869 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1870 1870 if (zio->io_error == 0) {
1871 1871 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
1872 1872 if (dr->dt.dl.dr_nopwrite) {
1873 1873 blkptr_t *bp = zio->io_bp;
1874 1874 blkptr_t *bp_orig = &zio->io_bp_orig;
1875 1875 uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
1876 1876
1877 1877 ASSERT(BP_EQUAL(bp, bp_orig));
1878 1878 VERIFY(BP_EQUAL(bp, db->db_blkptr));
1879 1879 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
1880 1880 ASSERT(zio_checksum_table[chksum].ci_flags &
1881 1881 ZCHECKSUM_FLAG_NOPWRITE);
1882 1882 }
1883 1883 dr->dt.dl.dr_overridden_by = *zio->io_bp;
1884 1884 dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1885 1885 dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
1886 1886
1887 1887 /*
1888 1888 * Old style holes are filled with all zeros, whereas
1889 1889 * new-style holes maintain their lsize, type, level,
1890 1890 * and birth time (see zio_write_compress). While we
1891 1891 * need to reset the BP_SET_LSIZE() call that happened
1892 1892 * in dmu_sync_ready for old style holes, we do *not*
1893 1893 * want to wipe out the information contained in new
1894 1894 * style holes. Thus, only zero out the block pointer if
1895 1895 * it's an old style hole.
1896 1896 */
1897 1897 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
1898 1898 dr->dt.dl.dr_overridden_by.blk_birth == 0)
1899 1899 BP_ZERO(&dr->dt.dl.dr_overridden_by);
1900 1900 } else {
1901 1901 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1902 1902 }
1903 1903 cv_broadcast(&db->db_changed);
1904 1904 mutex_exit(&db->db_mtx);
1905 1905
1906 1906 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1907 1907
1908 1908 kmem_free(dsa, sizeof (*dsa));
1909 1909 }
1910 1910
1911 1911 static void
1912 1912 dmu_sync_late_arrival_done(zio_t *zio)
1913 1913 {
1914 1914 blkptr_t *bp = zio->io_bp;
1915 1915 dmu_sync_arg_t *dsa = zio->io_private;
1916 1916 blkptr_t *bp_orig = &zio->io_bp_orig;
1917 1917 zgd_t *zgd = dsa->dsa_zgd;
1918 1918
1919 1919 if (zio->io_error == 0) {
1920 1920 /*
1921 1921 * Record the vdev(s) backing this blkptr so they can be
1922 1922 * flushed after the writes for the lwb have completed.
1923 1923 */
1924 1924 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1925 1925
1926 1926 if (!BP_IS_HOLE(bp)) {
1927 1927 ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE));
1928 1928 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
1929 1929 ASSERT(zio->io_bp->blk_birth == zio->io_txg);
1930 1930 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
1931 1931 zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
1932 1932 }
1933 1933 }
1934 1934
1935 1935 dmu_tx_commit(dsa->dsa_tx);
1936 1936
1937 1937 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1938 1938
1939 1939 abd_put(zio->io_abd);
1940 1940 kmem_free(dsa, sizeof (*dsa));
1941 1941 }
1942 1942
1943 1943 static int
1944 1944 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
1945 1945 zio_prop_t *zp, zbookmark_phys_t *zb)
1946 1946 {
1947 1947 dmu_sync_arg_t *dsa;
1948 1948 dmu_tx_t *tx;
1949 1949
1950 1950 tx = dmu_tx_create(os);
1951 1951 dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
1952 1952 if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1953 1953 dmu_tx_abort(tx);
1954 1954 /* Make zl_get_data do txg_waited_synced() */
1955 1955 return (SET_ERROR(EIO));
1956 1956 }
1957 1957
1958 1958 /*
1959 1959 * In order to prevent the zgd's lwb from being free'd prior to
1960 1960 * dmu_sync_late_arrival_done() being called, we have to ensure
1961 1961 * the lwb's "max txg" takes this tx's txg into account.
1962 1962 */
1963 1963 zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx));
1964 1964
1965 1965 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1966 1966 dsa->dsa_dr = NULL;
1967 1967 dsa->dsa_done = done;
1968 1968 dsa->dsa_zgd = zgd;
1969 1969 dsa->dsa_tx = tx;
1970 1970
1971 1971 /*
1972 1972 * Since we are currently syncing this txg, it's nontrivial to
1973 1973 * determine what BP to nopwrite against, so we disable nopwrite.
1974 1974 *
1975 1975 * When syncing, the db_blkptr is initially the BP of the previous
1976 1976 * txg. We can not nopwrite against it because it will be changed
1977 1977 * (this is similar to the non-late-arrival case where the dbuf is
1978 1978 * dirty in a future txg).
1979 1979 *
1980 1980 * Then dbuf_write_ready() sets bp_blkptr to the location we will write.
1981 1981 * We can not nopwrite against it because although the BP will not
1982 1982 * (typically) be changed, the data has not yet been persisted to this
1983 1983 * location.
1984 1984 *
1985 1985 * Finally, when dbuf_write_done() is called, it is theoretically
1986 1986 * possible to always nopwrite, because the data that was written in
1987 1987 * this txg is the same data that we are trying to write. However we
1988 1988 * would need to check that this dbuf is not dirty in any future
1989 1989 * txg's (as we do in the normal dmu_sync() path). For simplicity, we
1990 1990 * don't nopwrite in this case.
1991 1991 */
1992 1992 zp->zp_nopwrite = B_FALSE;
1993 1993
1994 1994 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
1995 1995 abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size),
1996 1996 zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp,
1997 1997 dmu_sync_late_arrival_ready, NULL, NULL, dmu_sync_late_arrival_done,
1998 1998 dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
1999 1999
2000 2000 return (0);
2001 2001 }
2002 2002
2003 2003 /*
2004 2004 * Intent log support: sync the block associated with db to disk.
2005 2005 * N.B. and XXX: the caller is responsible for making sure that the
2006 2006 * data isn't changing while dmu_sync() is writing it.
2007 2007 *
2008 2008 * Return values:
2009 2009 *
2010 2010 * EEXIST: this txg has already been synced, so there's nothing to do.
2011 2011 * The caller should not log the write.
2012 2012 *
2013 2013 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
2014 2014 * The caller should not log the write.
2015 2015 *
2016 2016 * EALREADY: this block is already in the process of being synced.
2017 2017 * The caller should track its progress (somehow).
2018 2018 *
2019 2019 * EIO: could not do the I/O.
2020 2020 * The caller should do a txg_wait_synced().
2021 2021 *
2022 2022 * 0: the I/O has been initiated.
2023 2023 * The caller should log this blkptr in the done callback.
2024 2024 * It is possible that the I/O will fail, in which case
2025 2025 * the error will be reported to the done callback and
2026 2026 * propagated to pio from zio_done().
2027 2027 */
2028 2028 int
2029 2029 dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
2030 2030 {
2031 2031 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
2032 2032 objset_t *os = db->db_objset;
2033 2033 dsl_dataset_t *ds = os->os_dsl_dataset;
2034 2034 dbuf_dirty_record_t *dr;
2035 2035 dmu_sync_arg_t *dsa;
2036 2036 zbookmark_phys_t zb;
2037 2037 zio_prop_t zp;
2038 2038 dnode_t *dn;
2039 2039
2040 2040 ASSERT(pio != NULL);
2041 2041 ASSERT(txg != 0);
2042 2042
2043 2043 SET_BOOKMARK(&zb, ds->ds_object,
2044 2044 db->db.db_object, db->db_level, db->db_blkid);
2045 2045
2046 2046 DB_DNODE_ENTER(db);
2047 2047 dn = DB_DNODE(db);
2048 2048 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
2049 2049 DB_DNODE_EXIT(db);
2050 2050
2051 2051 /*
2052 2052 * If we're frozen (running ziltest), we always need to generate a bp.
2053 2053 */
2054 2054 if (txg > spa_freeze_txg(os->os_spa))
2055 2055 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
2056 2056
2057 2057 /*
2058 2058 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
2059 2059 * and us. If we determine that this txg is not yet syncing,
2060 2060 * but it begins to sync a moment later, that's OK because the
2061 2061 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
2062 2062 */
2063 2063 mutex_enter(&db->db_mtx);
2064 2064
2065 2065 if (txg <= spa_last_synced_txg(os->os_spa)) {
2066 2066 /*
2067 2067 * This txg has already synced. There's nothing to do.
2068 2068 */
2069 2069 mutex_exit(&db->db_mtx);
2070 2070 return (SET_ERROR(EEXIST));
2071 2071 }
2072 2072
2073 2073 if (txg <= spa_syncing_txg(os->os_spa)) {
2074 2074 /*
2075 2075 * This txg is currently syncing, so we can't mess with
2076 2076 * the dirty record anymore; just write a new log block.
2077 2077 */
2078 2078 mutex_exit(&db->db_mtx);
2079 2079 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
2080 2080 }
2081 2081
2082 2082 dr = db->db_last_dirty;
2083 2083 while (dr && dr->dr_txg != txg)
2084 2084 dr = dr->dr_next;
2085 2085
2086 2086 if (dr == NULL) {
2087 2087 /*
2088 2088 * There's no dr for this dbuf, so it must have been freed.
2089 2089 * There's no need to log writes to freed blocks, so we're done.
2090 2090 */
2091 2091 mutex_exit(&db->db_mtx);
2092 2092 return (SET_ERROR(ENOENT));
2093 2093 }
2094 2094
2095 2095 ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
2096 2096
2097 2097 if (db->db_blkptr != NULL) {
2098 2098 /*
2099 2099 * We need to fill in zgd_bp with the current blkptr so that
2100 2100 * the nopwrite code can check if we're writing the same
2101 2101 * data that's already on disk. We can only nopwrite if we
2102 2102 * are sure that after making the copy, db_blkptr will not
2103 2103 * change until our i/o completes. We ensure this by
2104 2104 * holding the db_mtx, and only allowing nopwrite if the
2105 2105 * block is not already dirty (see below). This is verified
2106 2106 * by dmu_sync_done(), which VERIFYs that the db_blkptr has
2107 2107 * not changed.
2108 2108 */
2109 2109 *zgd->zgd_bp = *db->db_blkptr;
2110 2110 }
2111 2111
2112 2112 /*
2113 2113 * Assume the on-disk data is X, the current syncing data (in
2114 2114 * txg - 1) is Y, and the current in-memory data is Z (currently
2115 2115 * in dmu_sync).
2116 2116 *
2117 2117 * We usually want to perform a nopwrite if X and Z are the
2118 2118 * same. However, if Y is different (i.e. the BP is going to
2119 2119 * change before this write takes effect), then a nopwrite will
2120 2120 * be incorrect - we would override with X, which could have
2121 2121 * been freed when Y was written.
2122 2122 *
2123 2123 * (Note that this is not a concern when we are nop-writing from
2124 2124 * syncing context, because X and Y must be identical, because
2125 2125 * all previous txgs have been synced.)
2126 2126 *
2127 2127 * Therefore, we disable nopwrite if the current BP could change
2128 2128 * before this TXG. There are two ways it could change: by
2129 2129 * being dirty (dr_next is non-NULL), or by being freed
2130 2130 * (dnode_block_freed()). This behavior is verified by
2131 2131 * zio_done(), which VERIFYs that the override BP is identical
2132 2132 * to the on-disk BP.
2133 2133 */
2134 2134 DB_DNODE_ENTER(db);
2135 2135 dn = DB_DNODE(db);
2136 2136 if (dr->dr_next != NULL || dnode_block_freed(dn, db->db_blkid))
2137 2137 zp.zp_nopwrite = B_FALSE;
2138 2138 DB_DNODE_EXIT(db);
2139 2139
2140 2140 ASSERT(dr->dr_txg == txg);
2141 2141 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
2142 2142 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
2143 2143 /*
2144 2144 * We have already issued a sync write for this buffer,
2145 2145 * or this buffer has already been synced. It could not
2146 2146 * have been dirtied since, or we would have cleared the state.
2147 2147 */
2148 2148 mutex_exit(&db->db_mtx);
2149 2149 return (SET_ERROR(EALREADY));
2150 2150 }
2151 2151
2152 2152 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2153 2153 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
2154 2154 mutex_exit(&db->db_mtx);
2155 2155
2156 2156 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
2157 2157 dsa->dsa_dr = dr;
2158 2158 dsa->dsa_done = done;
2159 2159 dsa->dsa_zgd = zgd;
2160 2160 dsa->dsa_tx = NULL;
2161 2161
2162 2162 zio_nowait(arc_write(pio, os->os_spa, txg,
2163 2163 zgd->zgd_bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
2164 2164 &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa,
2165 2165 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
2166 2166
2167 2167 return (0);
2168 2168 }
2169 2169
2170 2170 int
2171 2171 dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels, dmu_tx_t *tx)
2172 2172 {
2173 2173 dnode_t *dn;
2174 2174 int err;
2175 2175
2176 2176 err = dnode_hold(os, object, FTAG, &dn);
2177 2177 if (err)
2178 2178 return (err);
2179 2179 err = dnode_set_nlevels(dn, nlevels, tx);
2180 2180 dnode_rele(dn, FTAG);
2181 2181 return (err);
2182 2182 }
2183 2183
2184 2184 int
2185 2185 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
2186 2186 dmu_tx_t *tx)
2187 2187 {
2188 2188 dnode_t *dn;
2189 2189 int err;
2190 2190
2191 2191 err = dnode_hold(os, object, FTAG, &dn);
2192 2192 if (err)
2193 2193 return (err);
2194 2194 err = dnode_set_blksz(dn, size, ibs, tx);
2195 2195 dnode_rele(dn, FTAG);
2196 2196 return (err);
2197 2197 }
2198 2198
2199 2199 int
2200 2200 dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid,
2201 2201 dmu_tx_t *tx)
2202 2202 {
2203 2203 dnode_t *dn;
2204 2204 int err;
2205 2205
2206 2206 err = dnode_hold(os, object, FTAG, &dn);
2207 2207 if (err)
2208 2208 return (err);
2209 2209 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2210 2210 dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE);
2211 2211 rw_exit(&dn->dn_struct_rwlock);
2212 2212 dnode_rele(dn, FTAG);
2213 2213 return (0);
2214 2214 }
2215 2215
2216 2216 void
2217 2217 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
2218 2218 dmu_tx_t *tx)
2219 2219 {
2220 2220 dnode_t *dn;
2221 2221
2222 2222 /*
2223 2223 * Send streams include each object's checksum function. This
2224 2224 * check ensures that the receiving system can understand the
2225 2225 * checksum function transmitted.
2226 2226 */
2227 2227 ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS);
2228 2228
2229 2229 VERIFY0(dnode_hold(os, object, FTAG, &dn));
2230 2230 ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS);
2231 2231 dn->dn_checksum = checksum;
2232 2232 dnode_setdirty(dn, tx);
2233 2233 dnode_rele(dn, FTAG);
2234 2234 }
2235 2235
2236 2236 void
2237 2237 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
2238 2238 dmu_tx_t *tx)
2239 2239 {
2240 2240 dnode_t *dn;
2241 2241
2242 2242 /*
2243 2243 * Send streams include each object's compression function. This
2244 2244 * check ensures that the receiving system can understand the
2245 2245 * compression function transmitted.
2246 2246 */
2247 2247 ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS);
2248 2248
2249 2249 VERIFY0(dnode_hold(os, object, FTAG, &dn));
2250 2250 dn->dn_compress = compress;
2251 2251 dnode_setdirty(dn, tx);
2252 2252 dnode_rele(dn, FTAG);
2253 2253 }
2254 2254
2255 2255 /*
2256 2256 * When the "redundant_metadata" property is set to "most", only indirect
2257 2257 * blocks of this level and higher will have an additional ditto block.
2258 2258 */
2259 2259 int zfs_redundant_metadata_most_ditto_level = 2;
2260 2260
2261 2261 void
2262 2262 dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
2263 2263 {
2264 2264 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
2265 2265 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
2266 2266 (wp & WP_SPILL));
2267 2267 enum zio_checksum checksum = os->os_checksum;
2268 2268 enum zio_compress compress = os->os_compress;
2269 2269 enum zio_checksum dedup_checksum = os->os_dedup_checksum;
2270 2270 boolean_t dedup = B_FALSE;
2271 2271 boolean_t nopwrite = B_FALSE;
2272 2272 boolean_t dedup_verify = os->os_dedup_verify;
2273 2273 boolean_t encrypt = B_FALSE;
2274 2274 int copies = os->os_copies;
2275 2275
2276 2276 /*
2277 2277 * We maintain different write policies for each of the following
2278 2278 * types of data:
2279 2279 * 1. metadata
2280 2280 * 2. preallocated blocks (i.e. level-0 blocks of a dump device)
2281 2281 * 3. all other level 0 blocks
2282 2282 */
2283 2283 if (ismd) {
2284 2284 /*
2285 2285 * XXX -- we should design a compression algorithm
2286 2286 * that specializes in arrays of bps.
2287 2287 */
2288 2288 compress = zio_compress_select(os->os_spa,
2289 2289 ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);
2290 2290
2291 2291 /*
2292 2292 * Metadata always gets checksummed. If the data
2293 2293 * checksum is multi-bit correctable, and it's not a
2294 2294 * ZBT-style checksum, then it's suitable for metadata
2295 2295 * as well. Otherwise, the metadata checksum defaults
2296 2296 * to fletcher4.
2297 2297 */
2298 2298 if (!(zio_checksum_table[checksum].ci_flags &
2299 2299 ZCHECKSUM_FLAG_METADATA) ||
2300 2300 (zio_checksum_table[checksum].ci_flags &
2301 2301 ZCHECKSUM_FLAG_EMBEDDED))
2302 2302 checksum = ZIO_CHECKSUM_FLETCHER_4;
2303 2303
2304 2304 if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL ||
2305 2305 (os->os_redundant_metadata ==
2306 2306 ZFS_REDUNDANT_METADATA_MOST &&
2307 2307 (level >= zfs_redundant_metadata_most_ditto_level ||
2308 2308 DMU_OT_IS_METADATA(type) || (wp & WP_SPILL))))
2309 2309 copies++;
2310 2310 } else if (wp & WP_NOFILL) {
2311 2311 ASSERT(level == 0);
2312 2312
2313 2313 /*
2314 2314 * If we're writing preallocated blocks, we aren't actually
2315 2315 * writing them so don't set any policy properties. These
2316 2316 * blocks are currently only used by an external subsystem
2317 2317 * outside of zfs (i.e. dump) and not written by the zio
2318 2318 * pipeline.
2319 2319 */
2320 2320 compress = ZIO_COMPRESS_OFF;
2321 2321 checksum = ZIO_CHECKSUM_NOPARITY;
2322 2322 } else {
2323 2323 compress = zio_compress_select(os->os_spa, dn->dn_compress,
2324 2324 compress);
2325 2325
2326 2326 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
2327 2327 zio_checksum_select(dn->dn_checksum, checksum) :
2328 2328 dedup_checksum;
2329 2329
2330 2330 /*
2331 2331 * Determine dedup setting. If we are in dmu_sync(),
2332 2332 * we won't actually dedup now because that's all
2333 2333 * done in syncing context; but we do want to use the
2334 2334 * dedup checkum. If the checksum is not strong
2335 2335 * enough to ensure unique signatures, force
2336 2336 * dedup_verify.
2337 2337 */
2338 2338 if (dedup_checksum != ZIO_CHECKSUM_OFF) {
2339 2339 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
2340 2340 if (!(zio_checksum_table[checksum].ci_flags &
2341 2341 ZCHECKSUM_FLAG_DEDUP))
2342 2342 dedup_verify = B_TRUE;
2343 2343 }
2344 2344
2345 2345 /*
2346 2346 * Enable nopwrite if we have secure enough checksum
2347 2347 * algorithm (see comment in zio_nop_write) and
2348 2348 * compression is enabled. We don't enable nopwrite if
2349 2349 * dedup is enabled as the two features are mutually
2350 2350 * exclusive.
2351 2351 */
2352 2352 nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
2353 2353 ZCHECKSUM_FLAG_NOPWRITE) &&
2354 2354 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
2355 2355 }
2356 2356
2357 2357 /*
2358 2358 * All objects in an encrypted objset are protected from modification
2359 2359 * via a MAC. Encrypted objects store their IV and salt in the last DVA
2360 2360 * in the bp, so we cannot use all copies. Encrypted objects are also
2361 2361 * not subject to nopwrite since writing the same data will still
2362 2362 * result in a new ciphertext. Only encrypted blocks can be dedup'd
2363 2363 * to avoid ambiguity in the dedup code since the DDT does not store
2364 2364 * object types.
2365 2365 */
2366 2366 if (os->os_encrypted && (wp & WP_NOFILL) == 0) {
2367 2367 encrypt = B_TRUE;
2368 2368
2369 2369 if (DMU_OT_IS_ENCRYPTED(type)) {
2370 2370 copies = MIN(copies, SPA_DVAS_PER_BP - 1);
2371 2371 nopwrite = B_FALSE;
2372 2372 } else {
2373 2373 dedup = B_FALSE;
2374 2374 }
2375 2375
2376 2376 if (level <= 0 &&
2377 2377 (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) {
2378 2378 compress = ZIO_COMPRESS_EMPTY;
2379 2379 }
2380 2380 }
2381 2381
2382 2382 zp->zp_compress = compress;
2383 2383 zp->zp_checksum = checksum;
2384 2384 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
2385 2385 zp->zp_level = level;
2386 2386 zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa));
2387 2387 zp->zp_dedup = dedup;
2388 2388 zp->zp_dedup_verify = dedup && dedup_verify;
2389 2389 zp->zp_nopwrite = nopwrite;
2390 2390 zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ?
2391 2391 os->os_zpl_special_smallblock : 0;
2392 2392 zp->zp_encrypt = encrypt;
2393 2393 zp->zp_byteorder = ZFS_HOST_BYTEORDER;
2394 2394 bzero(zp->zp_salt, ZIO_DATA_SALT_LEN);
2395 2395 bzero(zp->zp_iv, ZIO_DATA_IV_LEN);
2396 2396 bzero(zp->zp_mac, ZIO_DATA_MAC_LEN);
2397 2397 }
2398 2398
2399 2399 int
2400 2400 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
2401 2401 {
2402 2402 dnode_t *dn;
2403 2403 int err;
2404 2404
2405 2405 /*
2406 2406 * Sync any current changes before
2407 2407 * we go trundling through the block pointers.
2408 2408 */
2409 2409 err = dmu_object_wait_synced(os, object);
2410 2410 if (err) {
2411 2411 return (err);
2412 2412 }
2413 2413
2414 2414 err = dnode_hold(os, object, FTAG, &dn);
2415 2415 if (err) {
2416 2416 return (err);
2417 2417 }
2418 2418
2419 2419 err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
2420 2420 dnode_rele(dn, FTAG);
2421 2421
2422 2422 return (err);
2423 2423 }
2424 2424
2425 2425 /*
2426 2426 * Given the ZFS object, if it contains any dirty nodes
2427 2427 * this function flushes all dirty blocks to disk. This
2428 2428 * ensures the DMU object info is updated. A more efficient
2429 2429 * future version might just find the TXG with the maximum
2430 2430 * ID and wait for that to be synced.
2431 2431 */
2432 2432 int
2433 2433 dmu_object_wait_synced(objset_t *os, uint64_t object)
2434 2434 {
2435 2435 dnode_t *dn;
2436 2436 int error, i;
2437 2437
2438 2438 error = dnode_hold(os, object, FTAG, &dn);
2439 2439 if (error) {
2440 2440 return (error);
2441 2441 }
2442 2442
2443 2443 for (i = 0; i < TXG_SIZE; i++) {
2444 2444 if (list_link_active(&dn->dn_dirty_link[i])) {
2445 2445 break;
2446 2446 }
2447 2447 }
2448 2448 dnode_rele(dn, FTAG);
2449 2449 if (i != TXG_SIZE) {
2450 2450 txg_wait_synced(dmu_objset_pool(os), 0);
2451 2451 }
2452 2452
2453 2453 return (0);
2454 2454 }
2455 2455
2456 2456 void
2457 2457 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2458 2458 {
2459 2459 dnode_phys_t *dnp;
2460 2460
2461 2461 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2462 2462 mutex_enter(&dn->dn_mtx);
2463 2463
2464 2464 dnp = dn->dn_phys;
2465 2465
2466 2466 doi->doi_data_block_size = dn->dn_datablksz;
2467 2467 doi->doi_metadata_block_size = dn->dn_indblkshift ?
2468 2468 1ULL << dn->dn_indblkshift : 0;
2469 2469 doi->doi_type = dn->dn_type;
2470 2470 doi->doi_bonus_type = dn->dn_bonustype;
2471 2471 doi->doi_bonus_size = dn->dn_bonuslen;
2472 2472 doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT;
2473 2473 doi->doi_indirection = dn->dn_nlevels;
2474 2474 doi->doi_checksum = dn->dn_checksum;
2475 2475 doi->doi_compress = dn->dn_compress;
2476 2476 doi->doi_nblkptr = dn->dn_nblkptr;
2477 2477 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
2478 2478 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
2479 2479 doi->doi_fill_count = 0;
2480 2480 for (int i = 0; i < dnp->dn_nblkptr; i++)
2481 2481 doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]);
2482 2482
2483 2483 mutex_exit(&dn->dn_mtx);
2484 2484 rw_exit(&dn->dn_struct_rwlock);
2485 2485 }
2486 2486
2487 2487 /*
2488 2488 * Get information on a DMU object.
2489 2489 * If doi is NULL, just indicates whether the object exists.
2490 2490 */
2491 2491 int
2492 2492 dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
2493 2493 {
2494 2494 dnode_t *dn;
2495 2495 int err = dnode_hold(os, object, FTAG, &dn);
2496 2496
2497 2497 if (err)
2498 2498 return (err);
2499 2499
2500 2500 if (doi != NULL)
2501 2501 dmu_object_info_from_dnode(dn, doi);
2502 2502
2503 2503 dnode_rele(dn, FTAG);
2504 2504 return (0);
2505 2505 }
2506 2506
2507 2507 /*
2508 2508 * As above, but faster; can be used when you have a held dbuf in hand.
2509 2509 */
2510 2510 void
2511 2511 dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
2512 2512 {
2513 2513 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2514 2514
2515 2515 DB_DNODE_ENTER(db);
2516 2516 dmu_object_info_from_dnode(DB_DNODE(db), doi);
2517 2517 DB_DNODE_EXIT(db);
2518 2518 }
2519 2519
2520 2520 /*
2521 2521 * Faster still when you only care about the size.
2522 2522 * This is specifically optimized for zfs_getattr().
2523 2523 */
2524 2524 void
2525 2525 dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
2526 2526 u_longlong_t *nblk512)
2527 2527 {
2528 2528 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2529 2529 dnode_t *dn;
2530 2530
2531 2531 DB_DNODE_ENTER(db);
2532 2532 dn = DB_DNODE(db);
2533 2533
2534 2534 *blksize = dn->dn_datablksz;
2535 2535 /* add in number of slots used for the dnode itself */
2536 2536 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
2537 2537 SPA_MINBLOCKSHIFT) + dn->dn_num_slots;
2538 2538 DB_DNODE_EXIT(db);
2539 2539 }
2540 2540
2541 2541 void
2542 2542 dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize)
2543 2543 {
2544 2544 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2545 2545 dnode_t *dn;
2546 2546
2547 2547 DB_DNODE_ENTER(db);
2548 2548 dn = DB_DNODE(db);
2549 2549 *dnsize = dn->dn_num_slots << DNODE_SHIFT;
2550 2550 DB_DNODE_EXIT(db);
2551 2551 }
2552 2552
2553 2553 void
2554 2554 byteswap_uint64_array(void *vbuf, size_t size)
2555 2555 {
2556 2556 uint64_t *buf = vbuf;
2557 2557 size_t count = size >> 3;
2558 2558 int i;
2559 2559
2560 2560 ASSERT((size & 7) == 0);
2561 2561
2562 2562 for (i = 0; i < count; i++)
2563 2563 buf[i] = BSWAP_64(buf[i]);
2564 2564 }
2565 2565
2566 2566 void
2567 2567 byteswap_uint32_array(void *vbuf, size_t size)
2568 2568 {
2569 2569 uint32_t *buf = vbuf;
2570 2570 size_t count = size >> 2;
2571 2571 int i;
2572 2572
2573 2573 ASSERT((size & 3) == 0);
2574 2574
2575 2575 for (i = 0; i < count; i++)
2576 2576 buf[i] = BSWAP_32(buf[i]);
2577 2577 }
2578 2578
2579 2579 void
2580 2580 byteswap_uint16_array(void *vbuf, size_t size)
2581 2581 {
2582 2582 uint16_t *buf = vbuf;
2583 2583 size_t count = size >> 1;
2584 2584 int i;
2585 2585
2586 2586 ASSERT((size & 1) == 0);
2587 2587
2588 2588 for (i = 0; i < count; i++)
2589 2589 buf[i] = BSWAP_16(buf[i]);
2590 2590 }
2591 2591
2592 2592 /* ARGSUSED */
2593 2593 void
2594 2594 byteswap_uint8_array(void *vbuf, size_t size)
2595 2595 {
2596 2596 }
2597 2597
2598 2598 void
2599 2599 dmu_init(void)
2600 2600 {
2601 2601 abd_init();
2602 2602 zfs_dbgmsg_init();
2603 2603 sa_cache_init();
2604 2604 xuio_stat_init();
2605 2605 dmu_objset_init();
2606 2606 dnode_init();
2607 2607 zfetch_init();
2608 2608 l2arc_init();
2609 2609 arc_init();
2610 2610 dbuf_init();
2611 2611 }
2612 2612
2613 2613 void
2614 2614 dmu_fini(void)
2615 2615 {
2616 2616 arc_fini(); /* arc depends on l2arc, so arc must go first */
2617 2617 l2arc_fini();
2618 2618 zfetch_fini();
2619 2619 dbuf_fini();
2620 2620 dnode_fini();
2621 2621 dmu_objset_fini();
2622 2622 xuio_stat_fini();
2623 2623 sa_cache_fini();
2624 2624 zfs_dbgmsg_fini();
2625 2625 abd_fini();
2626 2626 }
|
↓ open down ↓ |
2254 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX