3756 want lz4 support for metadata compression
1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2013 by Delphix. All rights reserved. 25 * Copyright (c) 2013 Martin Matuska. All rights reserved. 26 */ 27 28 #include <sys/zfs_context.h> 29 #include <sys/dmu.h> 30 #include <sys/dmu_impl.h> 31 #include <sys/dbuf.h> 32 #include <sys/dmu_objset.h> 33 #include <sys/dsl_dataset.h> 34 #include <sys/dsl_dir.h> 35 #include <sys/dmu_tx.h> 36 #include <sys/spa.h> 37 #include <sys/zio.h> 38 #include <sys/dmu_zfetch.h> 39 #include <sys/sa.h> 40 #include <sys/sa_impl.h> 41 42 static void dbuf_destroy(dmu_buf_impl_t *db); 43 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 44 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 45 46 /* 47 * Global data structures and functions for the dbuf cache. 48 */ 49 static kmem_cache_t *dbuf_cache; 50 51 /* ARGSUSED */ 52 static int 53 dbuf_cons(void *vdb, void *unused, int kmflag) 54 { 55 dmu_buf_impl_t *db = vdb; 56 bzero(db, sizeof (dmu_buf_impl_t)); 57 58 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 59 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 60 refcount_create(&db->db_holds); 61 return (0); 62 } 63 64 /* ARGSUSED */ 65 static void 66 dbuf_dest(void *vdb, void *unused) 67 { 68 dmu_buf_impl_t *db = vdb; 69 mutex_destroy(&db->db_mtx); 70 cv_destroy(&db->db_changed); 71 refcount_destroy(&db->db_holds); 72 } 73 74 /* 75 * dbuf hash table routines 76 */ 77 static dbuf_hash_table_t dbuf_hash_table; 78 79 static uint64_t dbuf_hash_count; 80 81 static uint64_t 82 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 83 { 84 uintptr_t osv = (uintptr_t)os; 85 uint64_t crc = -1ULL; 86 87 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 88 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 89 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 90 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 91 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 92 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 93 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 94 95 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 96 97 return (crc); 98 } 99 100 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 101 102 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 103 ((dbuf)->db.db_object == (obj) && \ 104 (dbuf)->db_objset == (os) && \ 105 (dbuf)->db_level == (level) && \ 106 (dbuf)->db_blkid == (blkid)) 107 108 dmu_buf_impl_t * 109 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 110 { 111 dbuf_hash_table_t *h = &dbuf_hash_table; 112 objset_t *os = dn->dn_objset; 113 uint64_t obj = dn->dn_object; 114 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 115 uint64_t idx = hv & h->hash_table_mask; 116 dmu_buf_impl_t *db; 117 118 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 119 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 120 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 121 mutex_enter(&db->db_mtx); 122 if (db->db_state != DB_EVICTING) { 123 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 124 return (db); 125 } 126 mutex_exit(&db->db_mtx); 127 } 128 } 129 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 130 return (NULL); 131 } 132 133 /* 134 * Insert an entry into the hash table. If there is already an element 135 * equal to elem in the hash table, then the already existing element 136 * will be returned and the new element will not be inserted. 137 * Otherwise returns NULL. 138 */ 139 static dmu_buf_impl_t * 140 dbuf_hash_insert(dmu_buf_impl_t *db) 141 { 142 dbuf_hash_table_t *h = &dbuf_hash_table; 143 objset_t *os = db->db_objset; 144 uint64_t obj = db->db.db_object; 145 int level = db->db_level; 146 uint64_t blkid = db->db_blkid; 147 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 148 uint64_t idx = hv & h->hash_table_mask; 149 dmu_buf_impl_t *dbf; 150 151 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 152 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 153 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 154 mutex_enter(&dbf->db_mtx); 155 if (dbf->db_state != DB_EVICTING) { 156 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 157 return (dbf); 158 } 159 mutex_exit(&dbf->db_mtx); 160 } 161 } 162 163 mutex_enter(&db->db_mtx); 164 db->db_hash_next = h->hash_table[idx]; 165 h->hash_table[idx] = db; 166 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 167 atomic_add_64(&dbuf_hash_count, 1); 168 169 return (NULL); 170 } 171 172 /* 173 * Remove an entry from the hash table. This operation will 174 * fail if there are any existing holds on the db. 175 */ 176 static void 177 dbuf_hash_remove(dmu_buf_impl_t *db) 178 { 179 dbuf_hash_table_t *h = &dbuf_hash_table; 180 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 181 db->db_level, db->db_blkid); 182 uint64_t idx = hv & h->hash_table_mask; 183 dmu_buf_impl_t *dbf, **dbp; 184 185 /* 186 * We musn't hold db_mtx to maintin lock ordering: 187 * DBUF_HASH_MUTEX > db_mtx. 188 */ 189 ASSERT(refcount_is_zero(&db->db_holds)); 190 ASSERT(db->db_state == DB_EVICTING); 191 ASSERT(!MUTEX_HELD(&db->db_mtx)); 192 193 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 194 dbp = &h->hash_table[idx]; 195 while ((dbf = *dbp) != db) { 196 dbp = &dbf->db_hash_next; 197 ASSERT(dbf != NULL); 198 } 199 *dbp = db->db_hash_next; 200 db->db_hash_next = NULL; 201 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 202 atomic_add_64(&dbuf_hash_count, -1); 203 } 204 205 static arc_evict_func_t dbuf_do_evict; 206 207 static void 208 dbuf_evict_user(dmu_buf_impl_t *db) 209 { 210 ASSERT(MUTEX_HELD(&db->db_mtx)); 211 212 if (db->db_level != 0 || db->db_evict_func == NULL) 213 return; 214 215 if (db->db_user_data_ptr_ptr) 216 *db->db_user_data_ptr_ptr = db->db.db_data; 217 db->db_evict_func(&db->db, db->db_user_ptr); 218 db->db_user_ptr = NULL; 219 db->db_user_data_ptr_ptr = NULL; 220 db->db_evict_func = NULL; 221 } 222 223 boolean_t 224 dbuf_is_metadata(dmu_buf_impl_t *db) 225 { 226 if (db->db_level > 0) { 227 return (B_TRUE); 228 } else { 229 boolean_t is_metadata; 230 231 DB_DNODE_ENTER(db); 232 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 233 DB_DNODE_EXIT(db); 234 235 return (is_metadata); 236 } 237 } 238 239 void 240 dbuf_evict(dmu_buf_impl_t *db) 241 { 242 ASSERT(MUTEX_HELD(&db->db_mtx)); 243 ASSERT(db->db_buf == NULL); 244 ASSERT(db->db_data_pending == NULL); 245 246 dbuf_clear(db); 247 dbuf_destroy(db); 248 } 249 250 void 251 dbuf_init(void) 252 { 253 uint64_t hsize = 1ULL << 16; 254 dbuf_hash_table_t *h = &dbuf_hash_table; 255 int i; 256 257 /* 258 * The hash table is big enough to fill all of physical memory 259 * with an average 4K block size. The table will take up 260 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 261 */ 262 while (hsize * 4096 < physmem * PAGESIZE) 263 hsize <<= 1; 264 265 retry: 266 h->hash_table_mask = hsize - 1; 267 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 268 if (h->hash_table == NULL) { 269 /* XXX - we should really return an error instead of assert */ 270 ASSERT(hsize > (1ULL << 10)); 271 hsize >>= 1; 272 goto retry; 273 } 274 275 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 276 sizeof (dmu_buf_impl_t), 277 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 278 279 for (i = 0; i < DBUF_MUTEXES; i++) 280 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 281 } 282 283 void 284 dbuf_fini(void) 285 { 286 dbuf_hash_table_t *h = &dbuf_hash_table; 287 int i; 288 289 for (i = 0; i < DBUF_MUTEXES; i++) 290 mutex_destroy(&h->hash_mutexes[i]); 291 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 292 kmem_cache_destroy(dbuf_cache); 293 } 294 295 /* 296 * Other stuff. 297 */ 298 299 #ifdef ZFS_DEBUG 300 static void 301 dbuf_verify(dmu_buf_impl_t *db) 302 { 303 dnode_t *dn; 304 dbuf_dirty_record_t *dr; 305 306 ASSERT(MUTEX_HELD(&db->db_mtx)); 307 308 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 309 return; 310 311 ASSERT(db->db_objset != NULL); 312 DB_DNODE_ENTER(db); 313 dn = DB_DNODE(db); 314 if (dn == NULL) { 315 ASSERT(db->db_parent == NULL); 316 ASSERT(db->db_blkptr == NULL); 317 } else { 318 ASSERT3U(db->db.db_object, ==, dn->dn_object); 319 ASSERT3P(db->db_objset, ==, dn->dn_objset); 320 ASSERT3U(db->db_level, <, dn->dn_nlevels); 321 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 322 db->db_blkid == DMU_SPILL_BLKID || 323 !list_is_empty(&dn->dn_dbufs)); 324 } 325 if (db->db_blkid == DMU_BONUS_BLKID) { 326 ASSERT(dn != NULL); 327 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 328 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 329 } else if (db->db_blkid == DMU_SPILL_BLKID) { 330 ASSERT(dn != NULL); 331 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 332 ASSERT0(db->db.db_offset); 333 } else { 334 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 335 } 336 337 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 338 ASSERT(dr->dr_dbuf == db); 339 340 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 341 ASSERT(dr->dr_dbuf == db); 342 343 /* 344 * We can't assert that db_size matches dn_datablksz because it 345 * can be momentarily different when another thread is doing 346 * dnode_set_blksz(). 347 */ 348 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 349 dr = db->db_data_pending; 350 /* 351 * It should only be modified in syncing context, so 352 * make sure we only have one copy of the data. 353 */ 354 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 355 } 356 357 /* verify db->db_blkptr */ 358 if (db->db_blkptr) { 359 if (db->db_parent == dn->dn_dbuf) { 360 /* db is pointed to by the dnode */ 361 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 362 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 363 ASSERT(db->db_parent == NULL); 364 else 365 ASSERT(db->db_parent != NULL); 366 if (db->db_blkid != DMU_SPILL_BLKID) 367 ASSERT3P(db->db_blkptr, ==, 368 &dn->dn_phys->dn_blkptr[db->db_blkid]); 369 } else { 370 /* db is pointed to by an indirect block */ 371 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 372 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 373 ASSERT3U(db->db_parent->db.db_object, ==, 374 db->db.db_object); 375 /* 376 * dnode_grow_indblksz() can make this fail if we don't 377 * have the struct_rwlock. XXX indblksz no longer 378 * grows. safe to do this now? 379 */ 380 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 381 ASSERT3P(db->db_blkptr, ==, 382 ((blkptr_t *)db->db_parent->db.db_data + 383 db->db_blkid % epb)); 384 } 385 } 386 } 387 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 388 (db->db_buf == NULL || db->db_buf->b_data) && 389 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 390 db->db_state != DB_FILL && !dn->dn_free_txg) { 391 /* 392 * If the blkptr isn't set but they have nonzero data, 393 * it had better be dirty, otherwise we'll lose that 394 * data when we evict this buffer. 395 */ 396 if (db->db_dirtycnt == 0) { 397 uint64_t *buf = db->db.db_data; 398 int i; 399 400 for (i = 0; i < db->db.db_size >> 3; i++) { 401 ASSERT(buf[i] == 0); 402 } 403 } 404 } 405 DB_DNODE_EXIT(db); 406 } 407 #endif 408 409 static void 410 dbuf_update_data(dmu_buf_impl_t *db) 411 { 412 ASSERT(MUTEX_HELD(&db->db_mtx)); 413 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 414 ASSERT(!refcount_is_zero(&db->db_holds)); 415 *db->db_user_data_ptr_ptr = db->db.db_data; 416 } 417 } 418 419 static void 420 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 421 { 422 ASSERT(MUTEX_HELD(&db->db_mtx)); 423 ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 424 db->db_buf = buf; 425 if (buf != NULL) { 426 ASSERT(buf->b_data != NULL); 427 db->db.db_data = buf->b_data; 428 if (!arc_released(buf)) 429 arc_set_callback(buf, dbuf_do_evict, db); 430 dbuf_update_data(db); 431 } else { 432 dbuf_evict_user(db); 433 db->db.db_data = NULL; 434 if (db->db_state != DB_NOFILL) 435 db->db_state = DB_UNCACHED; 436 } 437 } 438 439 /* 440 * Loan out an arc_buf for read. Return the loaned arc_buf. 441 */ 442 arc_buf_t * 443 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 444 { 445 arc_buf_t *abuf; 446 447 mutex_enter(&db->db_mtx); 448 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 449 int blksz = db->db.db_size; 450 spa_t *spa; 451 452 mutex_exit(&db->db_mtx); 453 DB_GET_SPA(&spa, db); 454 abuf = arc_loan_buf(spa, blksz); 455 bcopy(db->db.db_data, abuf->b_data, blksz); 456 } else { 457 abuf = db->db_buf; 458 arc_loan_inuse_buf(abuf, db); 459 dbuf_set_data(db, NULL); 460 mutex_exit(&db->db_mtx); 461 } 462 return (abuf); 463 } 464 465 uint64_t 466 dbuf_whichblock(dnode_t *dn, uint64_t offset) 467 { 468 if (dn->dn_datablkshift) { 469 return (offset >> dn->dn_datablkshift); 470 } else { 471 ASSERT3U(offset, <, dn->dn_datablksz); 472 return (0); 473 } 474 } 475 476 static void 477 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 478 { 479 dmu_buf_impl_t *db = vdb; 480 481 mutex_enter(&db->db_mtx); 482 ASSERT3U(db->db_state, ==, DB_READ); 483 /* 484 * All reads are synchronous, so we must have a hold on the dbuf 485 */ 486 ASSERT(refcount_count(&db->db_holds) > 0); 487 ASSERT(db->db_buf == NULL); 488 ASSERT(db->db.db_data == NULL); 489 if (db->db_level == 0 && db->db_freed_in_flight) { 490 /* we were freed in flight; disregard any error */ 491 arc_release(buf, db); 492 bzero(buf->b_data, db->db.db_size); 493 arc_buf_freeze(buf); 494 db->db_freed_in_flight = FALSE; 495 dbuf_set_data(db, buf); 496 db->db_state = DB_CACHED; 497 } else if (zio == NULL || zio->io_error == 0) { 498 dbuf_set_data(db, buf); 499 db->db_state = DB_CACHED; 500 } else { 501 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 502 ASSERT3P(db->db_buf, ==, NULL); 503 VERIFY(arc_buf_remove_ref(buf, db)); 504 db->db_state = DB_UNCACHED; 505 } 506 cv_broadcast(&db->db_changed); 507 dbuf_rele_and_unlock(db, NULL); 508 } 509 510 static void 511 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 512 { 513 dnode_t *dn; 514 spa_t *spa; 515 zbookmark_t zb; 516 uint32_t aflags = ARC_NOWAIT; 517 518 DB_DNODE_ENTER(db); 519 dn = DB_DNODE(db); 520 ASSERT(!refcount_is_zero(&db->db_holds)); 521 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 522 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 523 ASSERT(MUTEX_HELD(&db->db_mtx)); 524 ASSERT(db->db_state == DB_UNCACHED); 525 ASSERT(db->db_buf == NULL); 526 527 if (db->db_blkid == DMU_BONUS_BLKID) { 528 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 529 530 ASSERT3U(bonuslen, <=, db->db.db_size); 531 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 532 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 533 if (bonuslen < DN_MAX_BONUSLEN) 534 bzero(db->db.db_data, DN_MAX_BONUSLEN); 535 if (bonuslen) 536 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 537 DB_DNODE_EXIT(db); 538 dbuf_update_data(db); 539 db->db_state = DB_CACHED; 540 mutex_exit(&db->db_mtx); 541 return; 542 } 543 544 /* 545 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 546 * processes the delete record and clears the bp while we are waiting 547 * for the dn_mtx (resulting in a "no" from block_freed). 548 */ 549 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 550 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 551 BP_IS_HOLE(db->db_blkptr)))) { 552 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 553 554 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 555 db->db.db_size, db, type)); 556 DB_DNODE_EXIT(db); 557 bzero(db->db.db_data, db->db.db_size); 558 db->db_state = DB_CACHED; 559 *flags |= DB_RF_CACHED; 560 mutex_exit(&db->db_mtx); 561 return; 562 } 563 564 spa = dn->dn_objset->os_spa; 565 DB_DNODE_EXIT(db); 566 567 db->db_state = DB_READ; 568 mutex_exit(&db->db_mtx); 569 570 if (DBUF_IS_L2CACHEABLE(db)) 571 aflags |= ARC_L2CACHE; 572 573 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 574 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 575 db->db.db_object, db->db_level, db->db_blkid); 576 577 dbuf_add_ref(db, NULL); 578 579 (void) arc_read(zio, spa, db->db_blkptr, 580 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 581 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 582 &aflags, &zb); 583 if (aflags & ARC_CACHED) 584 *flags |= DB_RF_CACHED; 585 } 586 587 int 588 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 589 { 590 int err = 0; 591 int havepzio = (zio != NULL); 592 int prefetch; 593 dnode_t *dn; 594 595 /* 596 * We don't have to hold the mutex to check db_state because it 597 * can't be freed while we have a hold on the buffer. 598 */ 599 ASSERT(!refcount_is_zero(&db->db_holds)); 600 601 if (db->db_state == DB_NOFILL) 602 return (SET_ERROR(EIO)); 603 604 DB_DNODE_ENTER(db); 605 dn = DB_DNODE(db); 606 if ((flags & DB_RF_HAVESTRUCT) == 0) 607 rw_enter(&dn->dn_struct_rwlock, RW_READER); 608 609 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 610 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 611 DBUF_IS_CACHEABLE(db); 612 613 mutex_enter(&db->db_mtx); 614 if (db->db_state == DB_CACHED) { 615 mutex_exit(&db->db_mtx); 616 if (prefetch) 617 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 618 db->db.db_size, TRUE); 619 if ((flags & DB_RF_HAVESTRUCT) == 0) 620 rw_exit(&dn->dn_struct_rwlock); 621 DB_DNODE_EXIT(db); 622 } else if (db->db_state == DB_UNCACHED) { 623 spa_t *spa = dn->dn_objset->os_spa; 624 625 if (zio == NULL) 626 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 627 dbuf_read_impl(db, zio, &flags); 628 629 /* dbuf_read_impl has dropped db_mtx for us */ 630 631 if (prefetch) 632 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 633 db->db.db_size, flags & DB_RF_CACHED); 634 635 if ((flags & DB_RF_HAVESTRUCT) == 0) 636 rw_exit(&dn->dn_struct_rwlock); 637 DB_DNODE_EXIT(db); 638 639 if (!havepzio) 640 err = zio_wait(zio); 641 } else { 642 mutex_exit(&db->db_mtx); 643 if (prefetch) 644 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 645 db->db.db_size, TRUE); 646 if ((flags & DB_RF_HAVESTRUCT) == 0) 647 rw_exit(&dn->dn_struct_rwlock); 648 DB_DNODE_EXIT(db); 649 650 mutex_enter(&db->db_mtx); 651 if ((flags & DB_RF_NEVERWAIT) == 0) { 652 while (db->db_state == DB_READ || 653 db->db_state == DB_FILL) { 654 ASSERT(db->db_state == DB_READ || 655 (flags & DB_RF_HAVESTRUCT) == 0); 656 cv_wait(&db->db_changed, &db->db_mtx); 657 } 658 if (db->db_state == DB_UNCACHED) 659 err = SET_ERROR(EIO); 660 } 661 mutex_exit(&db->db_mtx); 662 } 663 664 ASSERT(err || havepzio || db->db_state == DB_CACHED); 665 return (err); 666 } 667 668 static void 669 dbuf_noread(dmu_buf_impl_t *db) 670 { 671 ASSERT(!refcount_is_zero(&db->db_holds)); 672 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 673 mutex_enter(&db->db_mtx); 674 while (db->db_state == DB_READ || db->db_state == DB_FILL) 675 cv_wait(&db->db_changed, &db->db_mtx); 676 if (db->db_state == DB_UNCACHED) { 677 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 678 spa_t *spa; 679 680 ASSERT(db->db_buf == NULL); 681 ASSERT(db->db.db_data == NULL); 682 DB_GET_SPA(&spa, db); 683 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 684 db->db_state = DB_FILL; 685 } else if (db->db_state == DB_NOFILL) { 686 dbuf_set_data(db, NULL); 687 } else { 688 ASSERT3U(db->db_state, ==, DB_CACHED); 689 } 690 mutex_exit(&db->db_mtx); 691 } 692 693 /* 694 * This is our just-in-time copy function. It makes a copy of 695 * buffers, that have been modified in a previous transaction 696 * group, before we modify them in the current active group. 697 * 698 * This function is used in two places: when we are dirtying a 699 * buffer for the first time in a txg, and when we are freeing 700 * a range in a dnode that includes this buffer. 701 * 702 * Note that when we are called from dbuf_free_range() we do 703 * not put a hold on the buffer, we just traverse the active 704 * dbuf list for the dnode. 705 */ 706 static void 707 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 708 { 709 dbuf_dirty_record_t *dr = db->db_last_dirty; 710 711 ASSERT(MUTEX_HELD(&db->db_mtx)); 712 ASSERT(db->db.db_data != NULL); 713 ASSERT(db->db_level == 0); 714 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 715 716 if (dr == NULL || 717 (dr->dt.dl.dr_data != 718 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 719 return; 720 721 /* 722 * If the last dirty record for this dbuf has not yet synced 723 * and its referencing the dbuf data, either: 724 * reset the reference to point to a new copy, 725 * or (if there a no active holders) 726 * just null out the current db_data pointer. 727 */ 728 ASSERT(dr->dr_txg >= txg - 2); 729 if (db->db_blkid == DMU_BONUS_BLKID) { 730 /* Note that the data bufs here are zio_bufs */ 731 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 732 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 733 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 734 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 735 int size = db->db.db_size; 736 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 737 spa_t *spa; 738 739 DB_GET_SPA(&spa, db); 740 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 741 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 742 } else { 743 dbuf_set_data(db, NULL); 744 } 745 } 746 747 void 748 dbuf_unoverride(dbuf_dirty_record_t *dr) 749 { 750 dmu_buf_impl_t *db = dr->dr_dbuf; 751 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 752 uint64_t txg = dr->dr_txg; 753 754 ASSERT(MUTEX_HELD(&db->db_mtx)); 755 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 756 ASSERT(db->db_level == 0); 757 758 if (db->db_blkid == DMU_BONUS_BLKID || 759 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 760 return; 761 762 ASSERT(db->db_data_pending != dr); 763 764 /* free this block */ 765 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) { 766 spa_t *spa; 767 768 DB_GET_SPA(&spa, db); 769 zio_free(spa, txg, bp); 770 } 771 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 772 dr->dt.dl.dr_nopwrite = B_FALSE; 773 774 /* 775 * Release the already-written buffer, so we leave it in 776 * a consistent dirty state. Note that all callers are 777 * modifying the buffer, so they will immediately do 778 * another (redundant) arc_release(). Therefore, leave 779 * the buf thawed to save the effort of freezing & 780 * immediately re-thawing it. 781 */ 782 arc_release(dr->dt.dl.dr_data, db); 783 } 784 785 /* 786 * Evict (if its unreferenced) or clear (if its referenced) any level-0 787 * data blocks in the free range, so that any future readers will find 788 * empty blocks. Also, if we happen accross any level-1 dbufs in the 789 * range that have not already been marked dirty, mark them dirty so 790 * they stay in memory. 791 */ 792 void 793 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 794 { 795 dmu_buf_impl_t *db, *db_next; 796 uint64_t txg = tx->tx_txg; 797 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 798 uint64_t first_l1 = start >> epbs; 799 uint64_t last_l1 = end >> epbs; 800 801 if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 802 end = dn->dn_maxblkid; 803 last_l1 = end >> epbs; 804 } 805 dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 806 mutex_enter(&dn->dn_dbufs_mtx); 807 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 808 db_next = list_next(&dn->dn_dbufs, db); 809 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 810 811 if (db->db_level == 1 && 812 db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 813 mutex_enter(&db->db_mtx); 814 if (db->db_last_dirty && 815 db->db_last_dirty->dr_txg < txg) { 816 dbuf_add_ref(db, FTAG); 817 mutex_exit(&db->db_mtx); 818 dbuf_will_dirty(db, tx); 819 dbuf_rele(db, FTAG); 820 } else { 821 mutex_exit(&db->db_mtx); 822 } 823 } 824 825 if (db->db_level != 0) 826 continue; 827 dprintf_dbuf(db, "found buf %s\n", ""); 828 if (db->db_blkid < start || db->db_blkid > end) 829 continue; 830 831 /* found a level 0 buffer in the range */ 832 mutex_enter(&db->db_mtx); 833 if (dbuf_undirty(db, tx)) { 834 /* mutex has been dropped and dbuf destroyed */ 835 continue; 836 } 837 838 if (db->db_state == DB_UNCACHED || 839 db->db_state == DB_NOFILL || 840 db->db_state == DB_EVICTING) { 841 ASSERT(db->db.db_data == NULL); 842 mutex_exit(&db->db_mtx); 843 continue; 844 } 845 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 846 /* will be handled in dbuf_read_done or dbuf_rele */ 847 db->db_freed_in_flight = TRUE; 848 mutex_exit(&db->db_mtx); 849 continue; 850 } 851 if (refcount_count(&db->db_holds) == 0) { 852 ASSERT(db->db_buf); 853 dbuf_clear(db); 854 continue; 855 } 856 /* The dbuf is referenced */ 857 858 if (db->db_last_dirty != NULL) { 859 dbuf_dirty_record_t *dr = db->db_last_dirty; 860 861 if (dr->dr_txg == txg) { 862 /* 863 * This buffer is "in-use", re-adjust the file 864 * size to reflect that this buffer may 865 * contain new data when we sync. 866 */ 867 if (db->db_blkid != DMU_SPILL_BLKID && 868 db->db_blkid > dn->dn_maxblkid) 869 dn->dn_maxblkid = db->db_blkid; 870 dbuf_unoverride(dr); 871 } else { 872 /* 873 * This dbuf is not dirty in the open context. 874 * Either uncache it (if its not referenced in 875 * the open context) or reset its contents to 876 * empty. 877 */ 878 dbuf_fix_old_data(db, txg); 879 } 880 } 881 /* clear the contents if its cached */ 882 if (db->db_state == DB_CACHED) { 883 ASSERT(db->db.db_data != NULL); 884 arc_release(db->db_buf, db); 885 bzero(db->db.db_data, db->db.db_size); 886 arc_buf_freeze(db->db_buf); 887 } 888 889 mutex_exit(&db->db_mtx); 890 } 891 mutex_exit(&dn->dn_dbufs_mtx); 892 } 893 894 static int 895 dbuf_block_freeable(dmu_buf_impl_t *db) 896 { 897 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 898 uint64_t birth_txg = 0; 899 900 /* 901 * We don't need any locking to protect db_blkptr: 902 * If it's syncing, then db_last_dirty will be set 903 * so we'll ignore db_blkptr. 904 */ 905 ASSERT(MUTEX_HELD(&db->db_mtx)); 906 if (db->db_last_dirty) 907 birth_txg = db->db_last_dirty->dr_txg; 908 else if (db->db_blkptr) 909 birth_txg = db->db_blkptr->blk_birth; 910 911 /* 912 * If we don't exist or are in a snapshot, we can't be freed. 913 * Don't pass the bp to dsl_dataset_block_freeable() since we 914 * are holding the db_mtx lock and might deadlock if we are 915 * prefetching a dedup-ed block. 916 */ 917 if (birth_txg) 918 return (ds == NULL || 919 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 920 else 921 return (FALSE); 922 } 923 924 void 925 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 926 { 927 arc_buf_t *buf, *obuf; 928 int osize = db->db.db_size; 929 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 930 dnode_t *dn; 931 932 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 933 934 DB_DNODE_ENTER(db); 935 dn = DB_DNODE(db); 936 937 /* XXX does *this* func really need the lock? */ 938 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 939 940 /* 941 * This call to dbuf_will_dirty() with the dn_struct_rwlock held 942 * is OK, because there can be no other references to the db 943 * when we are changing its size, so no concurrent DB_FILL can 944 * be happening. 945 */ 946 /* 947 * XXX we should be doing a dbuf_read, checking the return 948 * value and returning that up to our callers 949 */ 950 dbuf_will_dirty(db, tx); 951 952 /* create the data buffer for the new block */ 953 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 954 955 /* copy old block data to the new block */ 956 obuf = db->db_buf; 957 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 958 /* zero the remainder */ 959 if (size > osize) 960 bzero((uint8_t *)buf->b_data + osize, size - osize); 961 962 mutex_enter(&db->db_mtx); 963 dbuf_set_data(db, buf); 964 VERIFY(arc_buf_remove_ref(obuf, db)); 965 db->db.db_size = size; 966 967 if (db->db_level == 0) { 968 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 969 db->db_last_dirty->dt.dl.dr_data = buf; 970 } 971 mutex_exit(&db->db_mtx); 972 973 dnode_willuse_space(dn, size-osize, tx); 974 DB_DNODE_EXIT(db); 975 } 976 977 void 978 dbuf_release_bp(dmu_buf_impl_t *db) 979 { 980 objset_t *os; 981 982 DB_GET_OBJSET(&os, db); 983 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 984 ASSERT(arc_released(os->os_phys_buf) || 985 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 986 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 987 988 (void) arc_release(db->db_buf, db); 989 } 990 991 dbuf_dirty_record_t * 992 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 993 { 994 dnode_t *dn; 995 objset_t *os; 996 dbuf_dirty_record_t **drp, *dr; 997 int drop_struct_lock = FALSE; 998 boolean_t do_free_accounting = B_FALSE; 999 int txgoff = tx->tx_txg & TXG_MASK; 1000 1001 ASSERT(tx->tx_txg != 0); 1002 ASSERT(!refcount_is_zero(&db->db_holds)); 1003 DMU_TX_DIRTY_BUF(tx, db); 1004 1005 DB_DNODE_ENTER(db); 1006 dn = DB_DNODE(db); 1007 /* 1008 * Shouldn't dirty a regular buffer in syncing context. Private 1009 * objects may be dirtied in syncing context, but only if they 1010 * were already pre-dirtied in open context. 1011 */ 1012 ASSERT(!dmu_tx_is_syncing(tx) || 1013 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1014 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1015 dn->dn_objset->os_dsl_dataset == NULL); 1016 /* 1017 * We make this assert for private objects as well, but after we 1018 * check if we're already dirty. They are allowed to re-dirty 1019 * in syncing context. 1020 */ 1021 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1022 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1023 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1024 1025 mutex_enter(&db->db_mtx); 1026 /* 1027 * XXX make this true for indirects too? The problem is that 1028 * transactions created with dmu_tx_create_assigned() from 1029 * syncing context don't bother holding ahead. 1030 */ 1031 ASSERT(db->db_level != 0 || 1032 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1033 db->db_state == DB_NOFILL); 1034 1035 mutex_enter(&dn->dn_mtx); 1036 /* 1037 * Don't set dirtyctx to SYNC if we're just modifying this as we 1038 * initialize the objset. 1039 */ 1040 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1041 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1042 dn->dn_dirtyctx = 1043 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1044 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1045 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1046 } 1047 mutex_exit(&dn->dn_mtx); 1048 1049 if (db->db_blkid == DMU_SPILL_BLKID) 1050 dn->dn_have_spill = B_TRUE; 1051 1052 /* 1053 * If this buffer is already dirty, we're done. 1054 */ 1055 drp = &db->db_last_dirty; 1056 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1057 db->db.db_object == DMU_META_DNODE_OBJECT); 1058 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1059 drp = &dr->dr_next; 1060 if (dr && dr->dr_txg == tx->tx_txg) { 1061 DB_DNODE_EXIT(db); 1062 1063 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1064 /* 1065 * If this buffer has already been written out, 1066 * we now need to reset its state. 1067 */ 1068 dbuf_unoverride(dr); 1069 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1070 db->db_state != DB_NOFILL) 1071 arc_buf_thaw(db->db_buf); 1072 } 1073 mutex_exit(&db->db_mtx); 1074 return (dr); 1075 } 1076 1077 /* 1078 * Only valid if not already dirty. 1079 */ 1080 ASSERT(dn->dn_object == 0 || 1081 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1082 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1083 1084 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1085 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1086 dn->dn_phys->dn_nlevels > db->db_level || 1087 dn->dn_next_nlevels[txgoff] > db->db_level || 1088 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1089 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1090 1091 /* 1092 * We should only be dirtying in syncing context if it's the 1093 * mos or we're initializing the os or it's a special object. 1094 * However, we are allowed to dirty in syncing context provided 1095 * we already dirtied it in open context. Hence we must make 1096 * this assertion only if we're not already dirty. 1097 */ 1098 os = dn->dn_objset; 1099 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1100 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1101 ASSERT(db->db.db_size != 0); 1102 1103 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1104 1105 if (db->db_blkid != DMU_BONUS_BLKID) { 1106 /* 1107 * Update the accounting. 1108 * Note: we delay "free accounting" until after we drop 1109 * the db_mtx. This keeps us from grabbing other locks 1110 * (and possibly deadlocking) in bp_get_dsize() while 1111 * also holding the db_mtx. 1112 */ 1113 dnode_willuse_space(dn, db->db.db_size, tx); 1114 do_free_accounting = dbuf_block_freeable(db); 1115 } 1116 1117 /* 1118 * If this buffer is dirty in an old transaction group we need 1119 * to make a copy of it so that the changes we make in this 1120 * transaction group won't leak out when we sync the older txg. 1121 */ 1122 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1123 if (db->db_level == 0) { 1124 void *data_old = db->db_buf; 1125 1126 if (db->db_state != DB_NOFILL) { 1127 if (db->db_blkid == DMU_BONUS_BLKID) { 1128 dbuf_fix_old_data(db, tx->tx_txg); 1129 data_old = db->db.db_data; 1130 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1131 /* 1132 * Release the data buffer from the cache so 1133 * that we can modify it without impacting 1134 * possible other users of this cached data 1135 * block. Note that indirect blocks and 1136 * private objects are not released until the 1137 * syncing state (since they are only modified 1138 * then). 1139 */ 1140 arc_release(db->db_buf, db); 1141 dbuf_fix_old_data(db, tx->tx_txg); 1142 data_old = db->db_buf; 1143 } 1144 ASSERT(data_old != NULL); 1145 } 1146 dr->dt.dl.dr_data = data_old; 1147 } else { 1148 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1149 list_create(&dr->dt.di.dr_children, 1150 sizeof (dbuf_dirty_record_t), 1151 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1152 } 1153 dr->dr_dbuf = db; 1154 dr->dr_txg = tx->tx_txg; 1155 dr->dr_next = *drp; 1156 *drp = dr; 1157 1158 /* 1159 * We could have been freed_in_flight between the dbuf_noread 1160 * and dbuf_dirty. We win, as though the dbuf_noread() had 1161 * happened after the free. 1162 */ 1163 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1164 db->db_blkid != DMU_SPILL_BLKID) { 1165 mutex_enter(&dn->dn_mtx); 1166 dnode_clear_range(dn, db->db_blkid, 1, tx); 1167 mutex_exit(&dn->dn_mtx); 1168 db->db_freed_in_flight = FALSE; 1169 } 1170 1171 /* 1172 * This buffer is now part of this txg 1173 */ 1174 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1175 db->db_dirtycnt += 1; 1176 ASSERT3U(db->db_dirtycnt, <=, 3); 1177 1178 mutex_exit(&db->db_mtx); 1179 1180 if (db->db_blkid == DMU_BONUS_BLKID || 1181 db->db_blkid == DMU_SPILL_BLKID) { 1182 mutex_enter(&dn->dn_mtx); 1183 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1184 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1185 mutex_exit(&dn->dn_mtx); 1186 dnode_setdirty(dn, tx); 1187 DB_DNODE_EXIT(db); 1188 return (dr); 1189 } else if (do_free_accounting) { 1190 blkptr_t *bp = db->db_blkptr; 1191 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1192 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1193 /* 1194 * This is only a guess -- if the dbuf is dirty 1195 * in a previous txg, we don't know how much 1196 * space it will use on disk yet. We should 1197 * really have the struct_rwlock to access 1198 * db_blkptr, but since this is just a guess, 1199 * it's OK if we get an odd answer. 1200 */ 1201 ddt_prefetch(os->os_spa, bp); 1202 dnode_willuse_space(dn, -willfree, tx); 1203 } 1204 1205 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1206 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1207 drop_struct_lock = TRUE; 1208 } 1209 1210 if (db->db_level == 0) { 1211 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1212 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1213 } 1214 1215 if (db->db_level+1 < dn->dn_nlevels) { 1216 dmu_buf_impl_t *parent = db->db_parent; 1217 dbuf_dirty_record_t *di; 1218 int parent_held = FALSE; 1219 1220 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1221 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1222 1223 parent = dbuf_hold_level(dn, db->db_level+1, 1224 db->db_blkid >> epbs, FTAG); 1225 ASSERT(parent != NULL); 1226 parent_held = TRUE; 1227 } 1228 if (drop_struct_lock) 1229 rw_exit(&dn->dn_struct_rwlock); 1230 ASSERT3U(db->db_level+1, ==, parent->db_level); 1231 di = dbuf_dirty(parent, tx); 1232 if (parent_held) 1233 dbuf_rele(parent, FTAG); 1234 1235 mutex_enter(&db->db_mtx); 1236 /* possible race with dbuf_undirty() */ 1237 if (db->db_last_dirty == dr || 1238 dn->dn_object == DMU_META_DNODE_OBJECT) { 1239 mutex_enter(&di->dt.di.dr_mtx); 1240 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1241 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1242 list_insert_tail(&di->dt.di.dr_children, dr); 1243 mutex_exit(&di->dt.di.dr_mtx); 1244 dr->dr_parent = di; 1245 } 1246 mutex_exit(&db->db_mtx); 1247 } else { 1248 ASSERT(db->db_level+1 == dn->dn_nlevels); 1249 ASSERT(db->db_blkid < dn->dn_nblkptr); 1250 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1251 mutex_enter(&dn->dn_mtx); 1252 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1253 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1254 mutex_exit(&dn->dn_mtx); 1255 if (drop_struct_lock) 1256 rw_exit(&dn->dn_struct_rwlock); 1257 } 1258 1259 dnode_setdirty(dn, tx); 1260 DB_DNODE_EXIT(db); 1261 return (dr); 1262 } 1263 1264 /* 1265 * Return TRUE if this evicted the dbuf. 1266 */ 1267 static boolean_t 1268 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1269 { 1270 dnode_t *dn; 1271 uint64_t txg = tx->tx_txg; 1272 dbuf_dirty_record_t *dr, **drp; 1273 1274 ASSERT(txg != 0); 1275 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1276 ASSERT0(db->db_level); 1277 ASSERT(MUTEX_HELD(&db->db_mtx)); 1278 1279 /* 1280 * If this buffer is not dirty, we're done. 1281 */ 1282 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1283 if (dr->dr_txg <= txg) 1284 break; 1285 if (dr == NULL || dr->dr_txg < txg) 1286 return (B_FALSE); 1287 ASSERT(dr->dr_txg == txg); 1288 ASSERT(dr->dr_dbuf == db); 1289 1290 DB_DNODE_ENTER(db); 1291 dn = DB_DNODE(db); 1292 1293 /* 1294 * Note: This code will probably work even if there are concurrent 1295 * holders, but it is untested in that scenerio, as the ZPL and 1296 * ztest have additional locking (the range locks) that prevents 1297 * that type of concurrent access. 1298 */ 1299 ASSERT3U(refcount_count(&db->db_holds), ==, db->db_dirtycnt); 1300 1301 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1302 1303 ASSERT(db->db.db_size != 0); 1304 1305 /* XXX would be nice to fix up dn_towrite_space[] */ 1306 1307 *drp = dr->dr_next; 1308 1309 /* 1310 * Note that there are three places in dbuf_dirty() 1311 * where this dirty record may be put on a list. 1312 * Make sure to do a list_remove corresponding to 1313 * every one of those list_insert calls. 1314 */ 1315 if (dr->dr_parent) { 1316 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1317 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1318 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1319 } else if (db->db_blkid == DMU_SPILL_BLKID || 1320 db->db_level+1 == dn->dn_nlevels) { 1321 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1322 mutex_enter(&dn->dn_mtx); 1323 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1324 mutex_exit(&dn->dn_mtx); 1325 } 1326 DB_DNODE_EXIT(db); 1327 1328 if (db->db_state != DB_NOFILL) { 1329 dbuf_unoverride(dr); 1330 1331 ASSERT(db->db_buf != NULL); 1332 ASSERT(dr->dt.dl.dr_data != NULL); 1333 if (dr->dt.dl.dr_data != db->db_buf) 1334 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1335 } 1336 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1337 1338 ASSERT(db->db_dirtycnt > 0); 1339 db->db_dirtycnt -= 1; 1340 1341 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1342 arc_buf_t *buf = db->db_buf; 1343 1344 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1345 dbuf_set_data(db, NULL); 1346 VERIFY(arc_buf_remove_ref(buf, db)); 1347 dbuf_evict(db); 1348 return (B_TRUE); 1349 } 1350 1351 return (B_FALSE); 1352 } 1353 1354 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1355 void 1356 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1357 { 1358 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1359 1360 ASSERT(tx->tx_txg != 0); 1361 ASSERT(!refcount_is_zero(&db->db_holds)); 1362 1363 DB_DNODE_ENTER(db); 1364 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1365 rf |= DB_RF_HAVESTRUCT; 1366 DB_DNODE_EXIT(db); 1367 (void) dbuf_read(db, NULL, rf); 1368 (void) dbuf_dirty(db, tx); 1369 } 1370 1371 void 1372 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1373 { 1374 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1375 1376 db->db_state = DB_NOFILL; 1377 1378 dmu_buf_will_fill(db_fake, tx); 1379 } 1380 1381 void 1382 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1383 { 1384 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1385 1386 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1387 ASSERT(tx->tx_txg != 0); 1388 ASSERT(db->db_level == 0); 1389 ASSERT(!refcount_is_zero(&db->db_holds)); 1390 1391 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1392 dmu_tx_private_ok(tx)); 1393 1394 dbuf_noread(db); 1395 (void) dbuf_dirty(db, tx); 1396 } 1397 1398 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1399 /* ARGSUSED */ 1400 void 1401 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1402 { 1403 mutex_enter(&db->db_mtx); 1404 DBUF_VERIFY(db); 1405 1406 if (db->db_state == DB_FILL) { 1407 if (db->db_level == 0 && db->db_freed_in_flight) { 1408 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1409 /* we were freed while filling */ 1410 /* XXX dbuf_undirty? */ 1411 bzero(db->db.db_data, db->db.db_size); 1412 db->db_freed_in_flight = FALSE; 1413 } 1414 db->db_state = DB_CACHED; 1415 cv_broadcast(&db->db_changed); 1416 } 1417 mutex_exit(&db->db_mtx); 1418 } 1419 1420 /* 1421 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1422 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1423 */ 1424 void 1425 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1426 { 1427 ASSERT(!refcount_is_zero(&db->db_holds)); 1428 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1429 ASSERT(db->db_level == 0); 1430 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1431 ASSERT(buf != NULL); 1432 ASSERT(arc_buf_size(buf) == db->db.db_size); 1433 ASSERT(tx->tx_txg != 0); 1434 1435 arc_return_buf(buf, db); 1436 ASSERT(arc_released(buf)); 1437 1438 mutex_enter(&db->db_mtx); 1439 1440 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1441 cv_wait(&db->db_changed, &db->db_mtx); 1442 1443 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1444 1445 if (db->db_state == DB_CACHED && 1446 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1447 mutex_exit(&db->db_mtx); 1448 (void) dbuf_dirty(db, tx); 1449 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1450 VERIFY(arc_buf_remove_ref(buf, db)); 1451 xuio_stat_wbuf_copied(); 1452 return; 1453 } 1454 1455 xuio_stat_wbuf_nocopy(); 1456 if (db->db_state == DB_CACHED) { 1457 dbuf_dirty_record_t *dr = db->db_last_dirty; 1458 1459 ASSERT(db->db_buf != NULL); 1460 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1461 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1462 if (!arc_released(db->db_buf)) { 1463 ASSERT(dr->dt.dl.dr_override_state == 1464 DR_OVERRIDDEN); 1465 arc_release(db->db_buf, db); 1466 } 1467 dr->dt.dl.dr_data = buf; 1468 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1469 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1470 arc_release(db->db_buf, db); 1471 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1472 } 1473 db->db_buf = NULL; 1474 } 1475 ASSERT(db->db_buf == NULL); 1476 dbuf_set_data(db, buf); 1477 db->db_state = DB_FILL; 1478 mutex_exit(&db->db_mtx); 1479 (void) dbuf_dirty(db, tx); 1480 dbuf_fill_done(db, tx); 1481 } 1482 1483 /* 1484 * "Clear" the contents of this dbuf. This will mark the dbuf 1485 * EVICTING and clear *most* of its references. Unfortunetely, 1486 * when we are not holding the dn_dbufs_mtx, we can't clear the 1487 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1488 * in this case. For callers from the DMU we will usually see: 1489 * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1490 * For the arc callback, we will usually see: 1491 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1492 * Sometimes, though, we will get a mix of these two: 1493 * DMU: dbuf_clear()->arc_buf_evict() 1494 * ARC: dbuf_do_evict()->dbuf_destroy() 1495 */ 1496 void 1497 dbuf_clear(dmu_buf_impl_t *db) 1498 { 1499 dnode_t *dn; 1500 dmu_buf_impl_t *parent = db->db_parent; 1501 dmu_buf_impl_t *dndb; 1502 int dbuf_gone = FALSE; 1503 1504 ASSERT(MUTEX_HELD(&db->db_mtx)); 1505 ASSERT(refcount_is_zero(&db->db_holds)); 1506 1507 dbuf_evict_user(db); 1508 1509 if (db->db_state == DB_CACHED) { 1510 ASSERT(db->db.db_data != NULL); 1511 if (db->db_blkid == DMU_BONUS_BLKID) { 1512 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1513 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1514 } 1515 db->db.db_data = NULL; 1516 db->db_state = DB_UNCACHED; 1517 } 1518 1519 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1520 ASSERT(db->db_data_pending == NULL); 1521 1522 db->db_state = DB_EVICTING; 1523 db->db_blkptr = NULL; 1524 1525 DB_DNODE_ENTER(db); 1526 dn = DB_DNODE(db); 1527 dndb = dn->dn_dbuf; 1528 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1529 list_remove(&dn->dn_dbufs, db); 1530 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1531 membar_producer(); 1532 DB_DNODE_EXIT(db); 1533 /* 1534 * Decrementing the dbuf count means that the hold corresponding 1535 * to the removed dbuf is no longer discounted in dnode_move(), 1536 * so the dnode cannot be moved until after we release the hold. 1537 * The membar_producer() ensures visibility of the decremented 1538 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1539 * release any lock. 1540 */ 1541 dnode_rele(dn, db); 1542 db->db_dnode_handle = NULL; 1543 } else { 1544 DB_DNODE_EXIT(db); 1545 } 1546 1547 if (db->db_buf) 1548 dbuf_gone = arc_buf_evict(db->db_buf); 1549 1550 if (!dbuf_gone) 1551 mutex_exit(&db->db_mtx); 1552 1553 /* 1554 * If this dbuf is referenced from an indirect dbuf, 1555 * decrement the ref count on the indirect dbuf. 1556 */ 1557 if (parent && parent != dndb) 1558 dbuf_rele(parent, db); 1559 } 1560 1561 static int 1562 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1563 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1564 { 1565 int nlevels, epbs; 1566 1567 *parentp = NULL; 1568 *bpp = NULL; 1569 1570 ASSERT(blkid != DMU_BONUS_BLKID); 1571 1572 if (blkid == DMU_SPILL_BLKID) { 1573 mutex_enter(&dn->dn_mtx); 1574 if (dn->dn_have_spill && 1575 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 1576 *bpp = &dn->dn_phys->dn_spill; 1577 else 1578 *bpp = NULL; 1579 dbuf_add_ref(dn->dn_dbuf, NULL); 1580 *parentp = dn->dn_dbuf; 1581 mutex_exit(&dn->dn_mtx); 1582 return (0); 1583 } 1584 1585 if (dn->dn_phys->dn_nlevels == 0) 1586 nlevels = 1; 1587 else 1588 nlevels = dn->dn_phys->dn_nlevels; 1589 1590 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1591 1592 ASSERT3U(level * epbs, <, 64); 1593 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1594 if (level >= nlevels || 1595 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1596 /* the buffer has no parent yet */ 1597 return (SET_ERROR(ENOENT)); 1598 } else if (level < nlevels-1) { 1599 /* this block is referenced from an indirect block */ 1600 int err = dbuf_hold_impl(dn, level+1, 1601 blkid >> epbs, fail_sparse, NULL, parentp); 1602 if (err) 1603 return (err); 1604 err = dbuf_read(*parentp, NULL, 1605 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1606 if (err) { 1607 dbuf_rele(*parentp, NULL); 1608 *parentp = NULL; 1609 return (err); 1610 } 1611 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1612 (blkid & ((1ULL << epbs) - 1)); 1613 return (0); 1614 } else { 1615 /* the block is referenced from the dnode */ 1616 ASSERT3U(level, ==, nlevels-1); 1617 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1618 blkid < dn->dn_phys->dn_nblkptr); 1619 if (dn->dn_dbuf) { 1620 dbuf_add_ref(dn->dn_dbuf, NULL); 1621 *parentp = dn->dn_dbuf; 1622 } 1623 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1624 return (0); 1625 } 1626 } 1627 1628 static dmu_buf_impl_t * 1629 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1630 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1631 { 1632 objset_t *os = dn->dn_objset; 1633 dmu_buf_impl_t *db, *odb; 1634 1635 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1636 ASSERT(dn->dn_type != DMU_OT_NONE); 1637 1638 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1639 1640 db->db_objset = os; 1641 db->db.db_object = dn->dn_object; 1642 db->db_level = level; 1643 db->db_blkid = blkid; 1644 db->db_last_dirty = NULL; 1645 db->db_dirtycnt = 0; 1646 db->db_dnode_handle = dn->dn_handle; 1647 db->db_parent = parent; 1648 db->db_blkptr = blkptr; 1649 1650 db->db_user_ptr = NULL; 1651 db->db_user_data_ptr_ptr = NULL; 1652 db->db_evict_func = NULL; 1653 db->db_immediate_evict = 0; 1654 db->db_freed_in_flight = 0; 1655 1656 if (blkid == DMU_BONUS_BLKID) { 1657 ASSERT3P(parent, ==, dn->dn_dbuf); 1658 db->db.db_size = DN_MAX_BONUSLEN - 1659 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1660 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1661 db->db.db_offset = DMU_BONUS_BLKID; 1662 db->db_state = DB_UNCACHED; 1663 /* the bonus dbuf is not placed in the hash table */ 1664 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1665 return (db); 1666 } else if (blkid == DMU_SPILL_BLKID) { 1667 db->db.db_size = (blkptr != NULL) ? 1668 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1669 db->db.db_offset = 0; 1670 } else { 1671 int blocksize = 1672 db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1673 db->db.db_size = blocksize; 1674 db->db.db_offset = db->db_blkid * blocksize; 1675 } 1676 1677 /* 1678 * Hold the dn_dbufs_mtx while we get the new dbuf 1679 * in the hash table *and* added to the dbufs list. 1680 * This prevents a possible deadlock with someone 1681 * trying to look up this dbuf before its added to the 1682 * dn_dbufs list. 1683 */ 1684 mutex_enter(&dn->dn_dbufs_mtx); 1685 db->db_state = DB_EVICTING; 1686 if ((odb = dbuf_hash_insert(db)) != NULL) { 1687 /* someone else inserted it first */ 1688 kmem_cache_free(dbuf_cache, db); 1689 mutex_exit(&dn->dn_dbufs_mtx); 1690 return (odb); 1691 } 1692 list_insert_head(&dn->dn_dbufs, db); 1693 db->db_state = DB_UNCACHED; 1694 mutex_exit(&dn->dn_dbufs_mtx); 1695 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1696 1697 if (parent && parent != dn->dn_dbuf) 1698 dbuf_add_ref(parent, db); 1699 1700 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1701 refcount_count(&dn->dn_holds) > 0); 1702 (void) refcount_add(&dn->dn_holds, db); 1703 (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1704 1705 dprintf_dbuf(db, "db=%p\n", db); 1706 1707 return (db); 1708 } 1709 1710 static int 1711 dbuf_do_evict(void *private) 1712 { 1713 arc_buf_t *buf = private; 1714 dmu_buf_impl_t *db = buf->b_private; 1715 1716 if (!MUTEX_HELD(&db->db_mtx)) 1717 mutex_enter(&db->db_mtx); 1718 1719 ASSERT(refcount_is_zero(&db->db_holds)); 1720 1721 if (db->db_state != DB_EVICTING) { 1722 ASSERT(db->db_state == DB_CACHED); 1723 DBUF_VERIFY(db); 1724 db->db_buf = NULL; 1725 dbuf_evict(db); 1726 } else { 1727 mutex_exit(&db->db_mtx); 1728 dbuf_destroy(db); 1729 } 1730 return (0); 1731 } 1732 1733 static void 1734 dbuf_destroy(dmu_buf_impl_t *db) 1735 { 1736 ASSERT(refcount_is_zero(&db->db_holds)); 1737 1738 if (db->db_blkid != DMU_BONUS_BLKID) { 1739 /* 1740 * If this dbuf is still on the dn_dbufs list, 1741 * remove it from that list. 1742 */ 1743 if (db->db_dnode_handle != NULL) { 1744 dnode_t *dn; 1745 1746 DB_DNODE_ENTER(db); 1747 dn = DB_DNODE(db); 1748 mutex_enter(&dn->dn_dbufs_mtx); 1749 list_remove(&dn->dn_dbufs, db); 1750 (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1751 mutex_exit(&dn->dn_dbufs_mtx); 1752 DB_DNODE_EXIT(db); 1753 /* 1754 * Decrementing the dbuf count means that the hold 1755 * corresponding to the removed dbuf is no longer 1756 * discounted in dnode_move(), so the dnode cannot be 1757 * moved until after we release the hold. 1758 */ 1759 dnode_rele(dn, db); 1760 db->db_dnode_handle = NULL; 1761 } 1762 dbuf_hash_remove(db); 1763 } 1764 db->db_parent = NULL; 1765 db->db_buf = NULL; 1766 1767 ASSERT(!list_link_active(&db->db_link)); 1768 ASSERT(db->db.db_data == NULL); 1769 ASSERT(db->db_hash_next == NULL); 1770 ASSERT(db->db_blkptr == NULL); 1771 ASSERT(db->db_data_pending == NULL); 1772 1773 kmem_cache_free(dbuf_cache, db); 1774 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1775 } 1776 1777 void 1778 dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1779 { 1780 dmu_buf_impl_t *db = NULL; 1781 blkptr_t *bp = NULL; 1782 1783 ASSERT(blkid != DMU_BONUS_BLKID); 1784 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1785 1786 if (dnode_block_freed(dn, blkid)) 1787 return; 1788 1789 /* dbuf_find() returns with db_mtx held */ 1790 if (db = dbuf_find(dn, 0, blkid)) { 1791 /* 1792 * This dbuf is already in the cache. We assume that 1793 * it is already CACHED, or else about to be either 1794 * read or filled. 1795 */ 1796 mutex_exit(&db->db_mtx); 1797 return; 1798 } 1799 1800 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1801 if (bp && !BP_IS_HOLE(bp)) { 1802 int priority = dn->dn_type == DMU_OT_DDT_ZAP ? 1803 ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ; 1804 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1805 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1806 zbookmark_t zb; 1807 1808 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1809 dn->dn_object, 0, blkid); 1810 1811 (void) arc_read(NULL, dn->dn_objset->os_spa, 1812 bp, NULL, NULL, priority, 1813 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1814 &aflags, &zb); 1815 } 1816 if (db) 1817 dbuf_rele(db, NULL); 1818 } 1819 } 1820 1821 /* 1822 * Returns with db_holds incremented, and db_mtx not held. 1823 * Note: dn_struct_rwlock must be held. 1824 */ 1825 int 1826 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1827 void *tag, dmu_buf_impl_t **dbp) 1828 { 1829 dmu_buf_impl_t *db, *parent = NULL; 1830 1831 ASSERT(blkid != DMU_BONUS_BLKID); 1832 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1833 ASSERT3U(dn->dn_nlevels, >, level); 1834 1835 *dbp = NULL; 1836 top: 1837 /* dbuf_find() returns with db_mtx held */ 1838 db = dbuf_find(dn, level, blkid); 1839 1840 if (db == NULL) { 1841 blkptr_t *bp = NULL; 1842 int err; 1843 1844 ASSERT3P(parent, ==, NULL); 1845 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1846 if (fail_sparse) { 1847 if (err == 0 && bp && BP_IS_HOLE(bp)) 1848 err = SET_ERROR(ENOENT); 1849 if (err) { 1850 if (parent) 1851 dbuf_rele(parent, NULL); 1852 return (err); 1853 } 1854 } 1855 if (err && err != ENOENT) 1856 return (err); 1857 db = dbuf_create(dn, level, blkid, parent, bp); 1858 } 1859 1860 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1861 arc_buf_add_ref(db->db_buf, db); 1862 if (db->db_buf->b_data == NULL) { 1863 dbuf_clear(db); 1864 if (parent) { 1865 dbuf_rele(parent, NULL); 1866 parent = NULL; 1867 } 1868 goto top; 1869 } 1870 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1871 } 1872 1873 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1874 1875 /* 1876 * If this buffer is currently syncing out, and we are are 1877 * still referencing it from db_data, we need to make a copy 1878 * of it in case we decide we want to dirty it again in this txg. 1879 */ 1880 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1881 dn->dn_object != DMU_META_DNODE_OBJECT && 1882 db->db_state == DB_CACHED && db->db_data_pending) { 1883 dbuf_dirty_record_t *dr = db->db_data_pending; 1884 1885 if (dr->dt.dl.dr_data == db->db_buf) { 1886 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1887 1888 dbuf_set_data(db, 1889 arc_buf_alloc(dn->dn_objset->os_spa, 1890 db->db.db_size, db, type)); 1891 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1892 db->db.db_size); 1893 } 1894 } 1895 1896 (void) refcount_add(&db->db_holds, tag); 1897 dbuf_update_data(db); 1898 DBUF_VERIFY(db); 1899 mutex_exit(&db->db_mtx); 1900 1901 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1902 if (parent) 1903 dbuf_rele(parent, NULL); 1904 1905 ASSERT3P(DB_DNODE(db), ==, dn); 1906 ASSERT3U(db->db_blkid, ==, blkid); 1907 ASSERT3U(db->db_level, ==, level); 1908 *dbp = db; 1909 1910 return (0); 1911 } 1912 1913 dmu_buf_impl_t * 1914 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1915 { 1916 dmu_buf_impl_t *db; 1917 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1918 return (err ? NULL : db); 1919 } 1920 1921 dmu_buf_impl_t * 1922 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1923 { 1924 dmu_buf_impl_t *db; 1925 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1926 return (err ? NULL : db); 1927 } 1928 1929 void 1930 dbuf_create_bonus(dnode_t *dn) 1931 { 1932 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1933 1934 ASSERT(dn->dn_bonus == NULL); 1935 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 1936 } 1937 1938 int 1939 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 1940 { 1941 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1942 dnode_t *dn; 1943 1944 if (db->db_blkid != DMU_SPILL_BLKID) 1945 return (SET_ERROR(ENOTSUP)); 1946 if (blksz == 0) 1947 blksz = SPA_MINBLOCKSIZE; 1948 if (blksz > SPA_MAXBLOCKSIZE) 1949 blksz = SPA_MAXBLOCKSIZE; 1950 else 1951 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 1952 1953 DB_DNODE_ENTER(db); 1954 dn = DB_DNODE(db); 1955 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1956 dbuf_new_size(db, blksz, tx); 1957 rw_exit(&dn->dn_struct_rwlock); 1958 DB_DNODE_EXIT(db); 1959 1960 return (0); 1961 } 1962 1963 void 1964 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 1965 { 1966 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 1967 } 1968 1969 #pragma weak dmu_buf_add_ref = dbuf_add_ref 1970 void 1971 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1972 { 1973 int64_t holds = refcount_add(&db->db_holds, tag); 1974 ASSERT(holds > 1); 1975 } 1976 1977 /* 1978 * If you call dbuf_rele() you had better not be referencing the dnode handle 1979 * unless you have some other direct or indirect hold on the dnode. (An indirect 1980 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 1981 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 1982 * dnode's parent dbuf evicting its dnode handles. 1983 */ 1984 #pragma weak dmu_buf_rele = dbuf_rele 1985 void 1986 dbuf_rele(dmu_buf_impl_t *db, void *tag) 1987 { 1988 mutex_enter(&db->db_mtx); 1989 dbuf_rele_and_unlock(db, tag); 1990 } 1991 1992 /* 1993 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 1994 * db_dirtycnt and db_holds to be updated atomically. 1995 */ 1996 void 1997 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 1998 { 1999 int64_t holds; 2000 2001 ASSERT(MUTEX_HELD(&db->db_mtx)); 2002 DBUF_VERIFY(db); 2003 2004 /* 2005 * Remove the reference to the dbuf before removing its hold on the 2006 * dnode so we can guarantee in dnode_move() that a referenced bonus 2007 * buffer has a corresponding dnode hold. 2008 */ 2009 holds = refcount_remove(&db->db_holds, tag); 2010 ASSERT(holds >= 0); 2011 2012 /* 2013 * We can't freeze indirects if there is a possibility that they 2014 * may be modified in the current syncing context. 2015 */ 2016 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 2017 arc_buf_freeze(db->db_buf); 2018 2019 if (holds == db->db_dirtycnt && 2020 db->db_level == 0 && db->db_immediate_evict) 2021 dbuf_evict_user(db); 2022 2023 if (holds == 0) { 2024 if (db->db_blkid == DMU_BONUS_BLKID) { 2025 mutex_exit(&db->db_mtx); 2026 2027 /* 2028 * If the dnode moves here, we cannot cross this barrier 2029 * until the move completes. 2030 */ 2031 DB_DNODE_ENTER(db); 2032 (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2033 DB_DNODE_EXIT(db); 2034 /* 2035 * The bonus buffer's dnode hold is no longer discounted 2036 * in dnode_move(). The dnode cannot move until after 2037 * the dnode_rele(). 2038 */ 2039 dnode_rele(DB_DNODE(db), db); 2040 } else if (db->db_buf == NULL) { 2041 /* 2042 * This is a special case: we never associated this 2043 * dbuf with any data allocated from the ARC. 2044 */ 2045 ASSERT(db->db_state == DB_UNCACHED || 2046 db->db_state == DB_NOFILL); 2047 dbuf_evict(db); 2048 } else if (arc_released(db->db_buf)) { 2049 arc_buf_t *buf = db->db_buf; 2050 /* 2051 * This dbuf has anonymous data associated with it. 2052 */ 2053 dbuf_set_data(db, NULL); 2054 VERIFY(arc_buf_remove_ref(buf, db)); 2055 dbuf_evict(db); 2056 } else { 2057 VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 2058 2059 /* 2060 * A dbuf will be eligible for eviction if either the 2061 * 'primarycache' property is set or a duplicate 2062 * copy of this buffer is already cached in the arc. 2063 * 2064 * In the case of the 'primarycache' a buffer 2065 * is considered for eviction if it matches the 2066 * criteria set in the property. 2067 * 2068 * To decide if our buffer is considered a 2069 * duplicate, we must call into the arc to determine 2070 * if multiple buffers are referencing the same 2071 * block on-disk. If so, then we simply evict 2072 * ourselves. 2073 */ 2074 if (!DBUF_IS_CACHEABLE(db) || 2075 arc_buf_eviction_needed(db->db_buf)) 2076 dbuf_clear(db); 2077 else 2078 mutex_exit(&db->db_mtx); 2079 } 2080 } else { 2081 mutex_exit(&db->db_mtx); 2082 } 2083 } 2084 2085 #pragma weak dmu_buf_refcount = dbuf_refcount 2086 uint64_t 2087 dbuf_refcount(dmu_buf_impl_t *db) 2088 { 2089 return (refcount_count(&db->db_holds)); 2090 } 2091 2092 void * 2093 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2094 dmu_buf_evict_func_t *evict_func) 2095 { 2096 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2097 user_data_ptr_ptr, evict_func)); 2098 } 2099 2100 void * 2101 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2102 dmu_buf_evict_func_t *evict_func) 2103 { 2104 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2105 2106 db->db_immediate_evict = TRUE; 2107 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2108 user_data_ptr_ptr, evict_func)); 2109 } 2110 2111 void * 2112 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2113 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2114 { 2115 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2116 ASSERT(db->db_level == 0); 2117 2118 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2119 2120 mutex_enter(&db->db_mtx); 2121 2122 if (db->db_user_ptr == old_user_ptr) { 2123 db->db_user_ptr = user_ptr; 2124 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2125 db->db_evict_func = evict_func; 2126 2127 dbuf_update_data(db); 2128 } else { 2129 old_user_ptr = db->db_user_ptr; 2130 } 2131 2132 mutex_exit(&db->db_mtx); 2133 return (old_user_ptr); 2134 } 2135 2136 void * 2137 dmu_buf_get_user(dmu_buf_t *db_fake) 2138 { 2139 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2140 ASSERT(!refcount_is_zero(&db->db_holds)); 2141 2142 return (db->db_user_ptr); 2143 } 2144 2145 boolean_t 2146 dmu_buf_freeable(dmu_buf_t *dbuf) 2147 { 2148 boolean_t res = B_FALSE; 2149 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2150 2151 if (db->db_blkptr) 2152 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2153 db->db_blkptr, db->db_blkptr->blk_birth); 2154 2155 return (res); 2156 } 2157 2158 blkptr_t * 2159 dmu_buf_get_blkptr(dmu_buf_t *db) 2160 { 2161 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 2162 return (dbi->db_blkptr); 2163 } 2164 2165 static void 2166 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2167 { 2168 /* ASSERT(dmu_tx_is_syncing(tx) */ 2169 ASSERT(MUTEX_HELD(&db->db_mtx)); 2170 2171 if (db->db_blkptr != NULL) 2172 return; 2173 2174 if (db->db_blkid == DMU_SPILL_BLKID) { 2175 db->db_blkptr = &dn->dn_phys->dn_spill; 2176 BP_ZERO(db->db_blkptr); 2177 return; 2178 } 2179 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2180 /* 2181 * This buffer was allocated at a time when there was 2182 * no available blkptrs from the dnode, or it was 2183 * inappropriate to hook it in (i.e., nlevels mis-match). 2184 */ 2185 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2186 ASSERT(db->db_parent == NULL); 2187 db->db_parent = dn->dn_dbuf; 2188 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2189 DBUF_VERIFY(db); 2190 } else { 2191 dmu_buf_impl_t *parent = db->db_parent; 2192 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2193 2194 ASSERT(dn->dn_phys->dn_nlevels > 1); 2195 if (parent == NULL) { 2196 mutex_exit(&db->db_mtx); 2197 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2198 (void) dbuf_hold_impl(dn, db->db_level+1, 2199 db->db_blkid >> epbs, FALSE, db, &parent); 2200 rw_exit(&dn->dn_struct_rwlock); 2201 mutex_enter(&db->db_mtx); 2202 db->db_parent = parent; 2203 } 2204 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2205 (db->db_blkid & ((1ULL << epbs) - 1)); 2206 DBUF_VERIFY(db); 2207 } 2208 } 2209 2210 static void 2211 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2212 { 2213 dmu_buf_impl_t *db = dr->dr_dbuf; 2214 dnode_t *dn; 2215 zio_t *zio; 2216 2217 ASSERT(dmu_tx_is_syncing(tx)); 2218 2219 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2220 2221 mutex_enter(&db->db_mtx); 2222 2223 ASSERT(db->db_level > 0); 2224 DBUF_VERIFY(db); 2225 2226 if (db->db_buf == NULL) { 2227 mutex_exit(&db->db_mtx); 2228 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2229 mutex_enter(&db->db_mtx); 2230 } 2231 ASSERT3U(db->db_state, ==, DB_CACHED); 2232 ASSERT(db->db_buf != NULL); 2233 2234 DB_DNODE_ENTER(db); 2235 dn = DB_DNODE(db); 2236 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2237 dbuf_check_blkptr(dn, db); 2238 DB_DNODE_EXIT(db); 2239 2240 db->db_data_pending = dr; 2241 2242 mutex_exit(&db->db_mtx); 2243 dbuf_write(dr, db->db_buf, tx); 2244 2245 zio = dr->dr_zio; 2246 mutex_enter(&dr->dt.di.dr_mtx); 2247 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2248 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2249 mutex_exit(&dr->dt.di.dr_mtx); 2250 zio_nowait(zio); 2251 } 2252 2253 static void 2254 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2255 { 2256 arc_buf_t **datap = &dr->dt.dl.dr_data; 2257 dmu_buf_impl_t *db = dr->dr_dbuf; 2258 dnode_t *dn; 2259 objset_t *os; 2260 uint64_t txg = tx->tx_txg; 2261 2262 ASSERT(dmu_tx_is_syncing(tx)); 2263 2264 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2265 2266 mutex_enter(&db->db_mtx); 2267 /* 2268 * To be synced, we must be dirtied. But we 2269 * might have been freed after the dirty. 2270 */ 2271 if (db->db_state == DB_UNCACHED) { 2272 /* This buffer has been freed since it was dirtied */ 2273 ASSERT(db->db.db_data == NULL); 2274 } else if (db->db_state == DB_FILL) { 2275 /* This buffer was freed and is now being re-filled */ 2276 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2277 } else { 2278 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2279 } 2280 DBUF_VERIFY(db); 2281 2282 DB_DNODE_ENTER(db); 2283 dn = DB_DNODE(db); 2284 2285 if (db->db_blkid == DMU_SPILL_BLKID) { 2286 mutex_enter(&dn->dn_mtx); 2287 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2288 mutex_exit(&dn->dn_mtx); 2289 } 2290 2291 /* 2292 * If this is a bonus buffer, simply copy the bonus data into the 2293 * dnode. It will be written out when the dnode is synced (and it 2294 * will be synced, since it must have been dirty for dbuf_sync to 2295 * be called). 2296 */ 2297 if (db->db_blkid == DMU_BONUS_BLKID) { 2298 dbuf_dirty_record_t **drp; 2299 2300 ASSERT(*datap != NULL); 2301 ASSERT0(db->db_level); 2302 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2303 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2304 DB_DNODE_EXIT(db); 2305 2306 if (*datap != db->db.db_data) { 2307 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2308 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2309 } 2310 db->db_data_pending = NULL; 2311 drp = &db->db_last_dirty; 2312 while (*drp != dr) 2313 drp = &(*drp)->dr_next; 2314 ASSERT(dr->dr_next == NULL); 2315 ASSERT(dr->dr_dbuf == db); 2316 *drp = dr->dr_next; 2317 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2318 ASSERT(db->db_dirtycnt > 0); 2319 db->db_dirtycnt -= 1; 2320 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2321 return; 2322 } 2323 2324 os = dn->dn_objset; 2325 2326 /* 2327 * This function may have dropped the db_mtx lock allowing a dmu_sync 2328 * operation to sneak in. As a result, we need to ensure that we 2329 * don't check the dr_override_state until we have returned from 2330 * dbuf_check_blkptr. 2331 */ 2332 dbuf_check_blkptr(dn, db); 2333 2334 /* 2335 * If this buffer is in the middle of an immediate write, 2336 * wait for the synchronous IO to complete. 2337 */ 2338 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2339 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2340 cv_wait(&db->db_changed, &db->db_mtx); 2341 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2342 } 2343 2344 if (db->db_state != DB_NOFILL && 2345 dn->dn_object != DMU_META_DNODE_OBJECT && 2346 refcount_count(&db->db_holds) > 1 && 2347 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2348 *datap == db->db_buf) { 2349 /* 2350 * If this buffer is currently "in use" (i.e., there 2351 * are active holds and db_data still references it), 2352 * then make a copy before we start the write so that 2353 * any modifications from the open txg will not leak 2354 * into this write. 2355 * 2356 * NOTE: this copy does not need to be made for 2357 * objects only modified in the syncing context (e.g. 2358 * DNONE_DNODE blocks). 2359 */ 2360 int blksz = arc_buf_size(*datap); 2361 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2362 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2363 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2364 } 2365 db->db_data_pending = dr; 2366 2367 mutex_exit(&db->db_mtx); 2368 2369 dbuf_write(dr, *datap, tx); 2370 2371 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2372 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2373 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2374 DB_DNODE_EXIT(db); 2375 } else { 2376 /* 2377 * Although zio_nowait() does not "wait for an IO", it does 2378 * initiate the IO. If this is an empty write it seems plausible 2379 * that the IO could actually be completed before the nowait 2380 * returns. We need to DB_DNODE_EXIT() first in case 2381 * zio_nowait() invalidates the dbuf. 2382 */ 2383 DB_DNODE_EXIT(db); 2384 zio_nowait(dr->dr_zio); 2385 } 2386 } 2387 2388 void 2389 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2390 { 2391 dbuf_dirty_record_t *dr; 2392 2393 while (dr = list_head(list)) { 2394 if (dr->dr_zio != NULL) { 2395 /* 2396 * If we find an already initialized zio then we 2397 * are processing the meta-dnode, and we have finished. 2398 * The dbufs for all dnodes are put back on the list 2399 * during processing, so that we can zio_wait() 2400 * these IOs after initiating all child IOs. 2401 */ 2402 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2403 DMU_META_DNODE_OBJECT); 2404 break; 2405 } 2406 list_remove(list, dr); 2407 if (dr->dr_dbuf->db_level > 0) 2408 dbuf_sync_indirect(dr, tx); 2409 else 2410 dbuf_sync_leaf(dr, tx); 2411 } 2412 } 2413 2414 /* ARGSUSED */ 2415 static void 2416 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2417 { 2418 dmu_buf_impl_t *db = vdb; 2419 dnode_t *dn; 2420 blkptr_t *bp = zio->io_bp; 2421 blkptr_t *bp_orig = &zio->io_bp_orig; 2422 spa_t *spa = zio->io_spa; 2423 int64_t delta; 2424 uint64_t fill = 0; 2425 int i; 2426 2427 ASSERT(db->db_blkptr == bp); 2428 2429 DB_DNODE_ENTER(db); 2430 dn = DB_DNODE(db); 2431 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2432 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2433 zio->io_prev_space_delta = delta; 2434 2435 if (BP_IS_HOLE(bp)) { 2436 ASSERT(bp->blk_fill == 0); 2437 DB_DNODE_EXIT(db); 2438 return; 2439 } 2440 2441 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2442 BP_GET_TYPE(bp) == dn->dn_type) || 2443 (db->db_blkid == DMU_SPILL_BLKID && 2444 BP_GET_TYPE(bp) == dn->dn_bonustype)); 2445 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2446 2447 mutex_enter(&db->db_mtx); 2448 2449 #ifdef ZFS_DEBUG 2450 if (db->db_blkid == DMU_SPILL_BLKID) { 2451 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2452 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2453 db->db_blkptr == &dn->dn_phys->dn_spill); 2454 } 2455 #endif 2456 2457 if (db->db_level == 0) { 2458 mutex_enter(&dn->dn_mtx); 2459 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2460 db->db_blkid != DMU_SPILL_BLKID) 2461 dn->dn_phys->dn_maxblkid = db->db_blkid; 2462 mutex_exit(&dn->dn_mtx); 2463 2464 if (dn->dn_type == DMU_OT_DNODE) { 2465 dnode_phys_t *dnp = db->db.db_data; 2466 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2467 i--, dnp++) { 2468 if (dnp->dn_type != DMU_OT_NONE) 2469 fill++; 2470 } 2471 } else { 2472 fill = 1; 2473 } 2474 } else { 2475 blkptr_t *ibp = db->db.db_data; 2476 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2477 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2478 if (BP_IS_HOLE(ibp)) 2479 continue; 2480 fill += ibp->blk_fill; 2481 } 2482 } 2483 DB_DNODE_EXIT(db); 2484 2485 bp->blk_fill = fill; 2486 2487 mutex_exit(&db->db_mtx); 2488 } 2489 2490 /* ARGSUSED */ 2491 static void 2492 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2493 { 2494 dmu_buf_impl_t *db = vdb; 2495 blkptr_t *bp = zio->io_bp; 2496 blkptr_t *bp_orig = &zio->io_bp_orig; 2497 uint64_t txg = zio->io_txg; 2498 dbuf_dirty_record_t **drp, *dr; 2499 2500 ASSERT0(zio->io_error); 2501 ASSERT(db->db_blkptr == bp); 2502 2503 /* 2504 * For nopwrites and rewrites we ensure that the bp matches our 2505 * original and bypass all the accounting. 2506 */ 2507 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2508 ASSERT(BP_EQUAL(bp, bp_orig)); 2509 } else { 2510 objset_t *os; 2511 dsl_dataset_t *ds; 2512 dmu_tx_t *tx; 2513 2514 DB_GET_OBJSET(&os, db); 2515 ds = os->os_dsl_dataset; 2516 tx = os->os_synctx; 2517 2518 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2519 dsl_dataset_block_born(ds, bp, tx); 2520 } 2521 2522 mutex_enter(&db->db_mtx); 2523 2524 DBUF_VERIFY(db); 2525 2526 drp = &db->db_last_dirty; 2527 while ((dr = *drp) != db->db_data_pending) 2528 drp = &dr->dr_next; 2529 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2530 ASSERT(dr->dr_txg == txg); 2531 ASSERT(dr->dr_dbuf == db); 2532 ASSERT(dr->dr_next == NULL); 2533 *drp = dr->dr_next; 2534 2535 #ifdef ZFS_DEBUG 2536 if (db->db_blkid == DMU_SPILL_BLKID) { 2537 dnode_t *dn; 2538 2539 DB_DNODE_ENTER(db); 2540 dn = DB_DNODE(db); 2541 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2542 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2543 db->db_blkptr == &dn->dn_phys->dn_spill); 2544 DB_DNODE_EXIT(db); 2545 } 2546 #endif 2547 2548 if (db->db_level == 0) { 2549 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2550 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2551 if (db->db_state != DB_NOFILL) { 2552 if (dr->dt.dl.dr_data != db->db_buf) 2553 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2554 db)); 2555 else if (!arc_released(db->db_buf)) 2556 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2557 } 2558 } else { 2559 dnode_t *dn; 2560 2561 DB_DNODE_ENTER(db); 2562 dn = DB_DNODE(db); 2563 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2564 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2565 if (!BP_IS_HOLE(db->db_blkptr)) { 2566 int epbs = 2567 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2568 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2569 db->db.db_size); 2570 ASSERT3U(dn->dn_phys->dn_maxblkid 2571 >> (db->db_level * epbs), >=, db->db_blkid); 2572 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2573 } 2574 DB_DNODE_EXIT(db); 2575 mutex_destroy(&dr->dt.di.dr_mtx); 2576 list_destroy(&dr->dt.di.dr_children); 2577 } 2578 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2579 2580 cv_broadcast(&db->db_changed); 2581 ASSERT(db->db_dirtycnt > 0); 2582 db->db_dirtycnt -= 1; 2583 db->db_data_pending = NULL; 2584 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2585 } 2586 2587 static void 2588 dbuf_write_nofill_ready(zio_t *zio) 2589 { 2590 dbuf_write_ready(zio, NULL, zio->io_private); 2591 } 2592 2593 static void 2594 dbuf_write_nofill_done(zio_t *zio) 2595 { 2596 dbuf_write_done(zio, NULL, zio->io_private); 2597 } 2598 2599 static void 2600 dbuf_write_override_ready(zio_t *zio) 2601 { 2602 dbuf_dirty_record_t *dr = zio->io_private; 2603 dmu_buf_impl_t *db = dr->dr_dbuf; 2604 2605 dbuf_write_ready(zio, NULL, db); 2606 } 2607 2608 static void 2609 dbuf_write_override_done(zio_t *zio) 2610 { 2611 dbuf_dirty_record_t *dr = zio->io_private; 2612 dmu_buf_impl_t *db = dr->dr_dbuf; 2613 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2614 2615 mutex_enter(&db->db_mtx); 2616 if (!BP_EQUAL(zio->io_bp, obp)) { 2617 if (!BP_IS_HOLE(obp)) 2618 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2619 arc_release(dr->dt.dl.dr_data, db); 2620 } 2621 mutex_exit(&db->db_mtx); 2622 2623 dbuf_write_done(zio, NULL, db); 2624 } 2625 2626 static void 2627 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2628 { 2629 dmu_buf_impl_t *db = dr->dr_dbuf; 2630 dnode_t *dn; 2631 objset_t *os; 2632 dmu_buf_impl_t *parent = db->db_parent; 2633 uint64_t txg = tx->tx_txg; 2634 zbookmark_t zb; 2635 zio_prop_t zp; 2636 zio_t *zio; 2637 int wp_flag = 0; 2638 2639 DB_DNODE_ENTER(db); 2640 dn = DB_DNODE(db); 2641 os = dn->dn_objset; 2642 2643 if (db->db_state != DB_NOFILL) { 2644 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2645 /* 2646 * Private object buffers are released here rather 2647 * than in dbuf_dirty() since they are only modified 2648 * in the syncing context and we don't want the 2649 * overhead of making multiple copies of the data. 2650 */ 2651 if (BP_IS_HOLE(db->db_blkptr)) { 2652 arc_buf_thaw(data); 2653 } else { 2654 dbuf_release_bp(db); 2655 } 2656 } 2657 } 2658 2659 if (parent != dn->dn_dbuf) { 2660 ASSERT(parent && parent->db_data_pending); 2661 ASSERT(db->db_level == parent->db_level-1); 2662 ASSERT(arc_released(parent->db_buf)); 2663 zio = parent->db_data_pending->dr_zio; 2664 } else { 2665 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2666 db->db_blkid != DMU_SPILL_BLKID) || 2667 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2668 if (db->db_blkid != DMU_SPILL_BLKID) 2669 ASSERT3P(db->db_blkptr, ==, 2670 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2671 zio = dn->dn_zio; 2672 } 2673 2674 ASSERT(db->db_level == 0 || data == db->db_buf); 2675 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2676 ASSERT(zio); 2677 2678 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2679 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2680 db->db.db_object, db->db_level, db->db_blkid); 2681 2682 if (db->db_blkid == DMU_SPILL_BLKID) 2683 wp_flag = WP_SPILL; 2684 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2685 2686 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp, txg); 2687 DB_DNODE_EXIT(db); 2688 2689 if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2690 ASSERT(db->db_state != DB_NOFILL); 2691 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2692 db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2693 dbuf_write_override_ready, dbuf_write_override_done, dr, 2694 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2695 mutex_enter(&db->db_mtx); 2696 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2697 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2698 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2699 mutex_exit(&db->db_mtx); 2700 } else if (db->db_state == DB_NOFILL) { 2701 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2702 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2703 db->db_blkptr, NULL, db->db.db_size, &zp, 2704 dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2705 ZIO_PRIORITY_ASYNC_WRITE, 2706 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2707 } else { 2708 ASSERT(arc_released(data)); 2709 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2710 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2711 dbuf_write_ready, dbuf_write_done, db, 2712 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2713 } 2714 } --- EOF ---