1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  24  * Copyright (c) 2012 by Delphix. All rights reserved.
  25  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  26  */
  27 
  28 #include <sys/zfs_context.h>
  29 #include <sys/dmu.h>
  30 #include <sys/dmu_impl.h>
  31 #include <sys/dbuf.h>
  32 #include <sys/dmu_objset.h>
  33 #include <sys/dsl_dataset.h>
  34 #include <sys/dsl_dir.h>
  35 #include <sys/dmu_tx.h>
  36 #include <sys/spa.h>
  37 #include <sys/zio.h>
  38 #include <sys/dmu_zfetch.h>
  39 #include <sys/sa.h>
  40 #include <sys/sa_impl.h>
  41 
  42 static void dbuf_destroy(dmu_buf_impl_t *db);
  43 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
  44 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
  45 
  46 /*
  47  * Global data structures and functions for the dbuf cache.
  48  */
  49 static kmem_cache_t *dbuf_cache;
  50 
  51 /* ARGSUSED */
  52 static int
  53 dbuf_cons(void *vdb, void *unused, int kmflag)
  54 {
  55         dmu_buf_impl_t *db = vdb;
  56         bzero(db, sizeof (dmu_buf_impl_t));
  57 
  58         mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
  59         cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
  60         refcount_create(&db->db_holds);
  61         return (0);
  62 }
  63 
  64 /* ARGSUSED */
  65 static void
  66 dbuf_dest(void *vdb, void *unused)
  67 {
  68         dmu_buf_impl_t *db = vdb;
  69         mutex_destroy(&db->db_mtx);
  70         cv_destroy(&db->db_changed);
  71         refcount_destroy(&db->db_holds);
  72 }
  73 
  74 /*
  75  * dbuf hash table routines
  76  */
  77 static dbuf_hash_table_t dbuf_hash_table;
  78 
  79 static uint64_t dbuf_hash_count;
  80 
  81 static uint64_t
  82 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
  83 {
  84         uintptr_t osv = (uintptr_t)os;
  85         uint64_t crc = -1ULL;
  86 
  87         ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
  88         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
  89         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
  90         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
  91         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
  92         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
  93         crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
  94 
  95         crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
  96 
  97         return (crc);
  98 }
  99 
 100 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
 101 
 102 #define DBUF_EQUAL(dbuf, os, obj, level, blkid)         \
 103         ((dbuf)->db.db_object == (obj) &&            \
 104         (dbuf)->db_objset == (os) &&                 \
 105         (dbuf)->db_level == (level) &&                       \
 106         (dbuf)->db_blkid == (blkid))
 107 
 108 dmu_buf_impl_t *
 109 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
 110 {
 111         dbuf_hash_table_t *h = &dbuf_hash_table;
 112         objset_t *os = dn->dn_objset;
 113         uint64_t obj = dn->dn_object;
 114         uint64_t hv = DBUF_HASH(os, obj, level, blkid);
 115         uint64_t idx = hv & h->hash_table_mask;
 116         dmu_buf_impl_t *db;
 117 
 118         mutex_enter(DBUF_HASH_MUTEX(h, idx));
 119         for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
 120                 if (DBUF_EQUAL(db, os, obj, level, blkid)) {
 121                         mutex_enter(&db->db_mtx);
 122                         if (db->db_state != DB_EVICTING) {
 123                                 mutex_exit(DBUF_HASH_MUTEX(h, idx));
 124                                 return (db);
 125                         }
 126                         mutex_exit(&db->db_mtx);
 127                 }
 128         }
 129         mutex_exit(DBUF_HASH_MUTEX(h, idx));
 130         return (NULL);
 131 }
 132 
 133 /*
 134  * Insert an entry into the hash table.  If there is already an element
 135  * equal to elem in the hash table, then the already existing element
 136  * will be returned and the new element will not be inserted.
 137  * Otherwise returns NULL.
 138  */
 139 static dmu_buf_impl_t *
 140 dbuf_hash_insert(dmu_buf_impl_t *db)
 141 {
 142         dbuf_hash_table_t *h = &dbuf_hash_table;
 143         objset_t *os = db->db_objset;
 144         uint64_t obj = db->db.db_object;
 145         int level = db->db_level;
 146         uint64_t blkid = db->db_blkid;
 147         uint64_t hv = DBUF_HASH(os, obj, level, blkid);
 148         uint64_t idx = hv & h->hash_table_mask;
 149         dmu_buf_impl_t *dbf;
 150 
 151         mutex_enter(DBUF_HASH_MUTEX(h, idx));
 152         for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
 153                 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
 154                         mutex_enter(&dbf->db_mtx);
 155                         if (dbf->db_state != DB_EVICTING) {
 156                                 mutex_exit(DBUF_HASH_MUTEX(h, idx));
 157                                 return (dbf);
 158                         }
 159                         mutex_exit(&dbf->db_mtx);
 160                 }
 161         }
 162 
 163         mutex_enter(&db->db_mtx);
 164         db->db_hash_next = h->hash_table[idx];
 165         h->hash_table[idx] = db;
 166         mutex_exit(DBUF_HASH_MUTEX(h, idx));
 167         atomic_add_64(&dbuf_hash_count, 1);
 168 
 169         return (NULL);
 170 }
 171 
 172 /*
 173  * Remove an entry from the hash table.  This operation will
 174  * fail if there are any existing holds on the db.
 175  */
 176 static void
 177 dbuf_hash_remove(dmu_buf_impl_t *db)
 178 {
 179         dbuf_hash_table_t *h = &dbuf_hash_table;
 180         uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
 181             db->db_level, db->db_blkid);
 182         uint64_t idx = hv & h->hash_table_mask;
 183         dmu_buf_impl_t *dbf, **dbp;
 184 
 185         /*
 186          * We musn't hold db_mtx to maintin lock ordering:
 187          * DBUF_HASH_MUTEX > db_mtx.
 188          */
 189         ASSERT(refcount_is_zero(&db->db_holds));
 190         ASSERT(db->db_state == DB_EVICTING);
 191         ASSERT(!MUTEX_HELD(&db->db_mtx));
 192 
 193         mutex_enter(DBUF_HASH_MUTEX(h, idx));
 194         dbp = &h->hash_table[idx];
 195         while ((dbf = *dbp) != db) {
 196                 dbp = &dbf->db_hash_next;
 197                 ASSERT(dbf != NULL);
 198         }
 199         *dbp = db->db_hash_next;
 200         db->db_hash_next = NULL;
 201         mutex_exit(DBUF_HASH_MUTEX(h, idx));
 202         atomic_add_64(&dbuf_hash_count, -1);
 203 }
 204 
 205 static arc_evict_func_t dbuf_do_evict;
 206 
 207 static void
 208 dbuf_evict_user(dmu_buf_impl_t *db)
 209 {
 210         ASSERT(MUTEX_HELD(&db->db_mtx));
 211 
 212         if (db->db_level != 0 || db->db_evict_func == NULL)
 213                 return;
 214 
 215         if (db->db_user_data_ptr_ptr)
 216                 *db->db_user_data_ptr_ptr = db->db.db_data;
 217         db->db_evict_func(&db->db, db->db_user_ptr);
 218         db->db_user_ptr = NULL;
 219         db->db_user_data_ptr_ptr = NULL;
 220         db->db_evict_func = NULL;
 221 }
 222 
 223 boolean_t
 224 dbuf_is_metadata(dmu_buf_impl_t *db)
 225 {
 226         if (db->db_level > 0) {
 227                 return (B_TRUE);
 228         } else {
 229                 boolean_t is_metadata;
 230 
 231                 DB_DNODE_ENTER(db);
 232                 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
 233                 DB_DNODE_EXIT(db);
 234 
 235                 return (is_metadata);
 236         }
 237 }
 238 
 239 void
 240 dbuf_evict(dmu_buf_impl_t *db)
 241 {
 242         ASSERT(MUTEX_HELD(&db->db_mtx));
 243         ASSERT(db->db_buf == NULL);
 244         ASSERT(db->db_data_pending == NULL);
 245 
 246         dbuf_clear(db);
 247         dbuf_destroy(db);
 248 }
 249 
 250 void
 251 dbuf_init(void)
 252 {
 253         uint64_t hsize = 1ULL << 16;
 254         dbuf_hash_table_t *h = &dbuf_hash_table;
 255         int i;
 256 
 257         /*
 258          * The hash table is big enough to fill all of physical memory
 259          * with an average 4K block size.  The table will take up
 260          * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
 261          */
 262         while (hsize * 4096 < physmem * PAGESIZE)
 263                 hsize <<= 1;
 264 
 265 retry:
 266         h->hash_table_mask = hsize - 1;
 267         h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
 268         if (h->hash_table == NULL) {
 269                 /* XXX - we should really return an error instead of assert */
 270                 ASSERT(hsize > (1ULL << 10));
 271                 hsize >>= 1;
 272                 goto retry;
 273         }
 274 
 275         dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
 276             sizeof (dmu_buf_impl_t),
 277             0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
 278 
 279         for (i = 0; i < DBUF_MUTEXES; i++)
 280                 mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
 281 }
 282 
 283 void
 284 dbuf_fini(void)
 285 {
 286         dbuf_hash_table_t *h = &dbuf_hash_table;
 287         int i;
 288 
 289         for (i = 0; i < DBUF_MUTEXES; i++)
 290                 mutex_destroy(&h->hash_mutexes[i]);
 291         kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
 292         kmem_cache_destroy(dbuf_cache);
 293 }
 294 
 295 /*
 296  * Other stuff.
 297  */
 298 
 299 #ifdef ZFS_DEBUG
 300 static void
 301 dbuf_verify(dmu_buf_impl_t *db)
 302 {
 303         dnode_t *dn;
 304         dbuf_dirty_record_t *dr;
 305 
 306         ASSERT(MUTEX_HELD(&db->db_mtx));
 307 
 308         if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
 309                 return;
 310 
 311         ASSERT(db->db_objset != NULL);
 312         DB_DNODE_ENTER(db);
 313         dn = DB_DNODE(db);
 314         if (dn == NULL) {
 315                 ASSERT(db->db_parent == NULL);
 316                 ASSERT(db->db_blkptr == NULL);
 317         } else {
 318                 ASSERT3U(db->db.db_object, ==, dn->dn_object);
 319                 ASSERT3P(db->db_objset, ==, dn->dn_objset);
 320                 ASSERT3U(db->db_level, <, dn->dn_nlevels);
 321                 ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
 322                     db->db_blkid == DMU_SPILL_BLKID ||
 323                     !list_is_empty(&dn->dn_dbufs));
 324         }
 325         if (db->db_blkid == DMU_BONUS_BLKID) {
 326                 ASSERT(dn != NULL);
 327                 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
 328                 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
 329         } else if (db->db_blkid == DMU_SPILL_BLKID) {
 330                 ASSERT(dn != NULL);
 331                 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
 332                 ASSERT0(db->db.db_offset);
 333         } else {
 334                 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
 335         }
 336 
 337         for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
 338                 ASSERT(dr->dr_dbuf == db);
 339 
 340         for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
 341                 ASSERT(dr->dr_dbuf == db);
 342 
 343         /*
 344          * We can't assert that db_size matches dn_datablksz because it
 345          * can be momentarily different when another thread is doing
 346          * dnode_set_blksz().
 347          */
 348         if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
 349                 dr = db->db_data_pending;
 350                 /*
 351                  * It should only be modified in syncing context, so
 352                  * make sure we only have one copy of the data.
 353                  */
 354                 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
 355         }
 356 
 357         /* verify db->db_blkptr */
 358         if (db->db_blkptr) {
 359                 if (db->db_parent == dn->dn_dbuf) {
 360                         /* db is pointed to by the dnode */
 361                         /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
 362                         if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
 363                                 ASSERT(db->db_parent == NULL);
 364                         else
 365                                 ASSERT(db->db_parent != NULL);
 366                         if (db->db_blkid != DMU_SPILL_BLKID)
 367                                 ASSERT3P(db->db_blkptr, ==,
 368                                     &dn->dn_phys->dn_blkptr[db->db_blkid]);
 369                 } else {
 370                         /* db is pointed to by an indirect block */
 371                         int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
 372                         ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
 373                         ASSERT3U(db->db_parent->db.db_object, ==,
 374                             db->db.db_object);
 375                         /*
 376                          * dnode_grow_indblksz() can make this fail if we don't
 377                          * have the struct_rwlock.  XXX indblksz no longer
 378                          * grows.  safe to do this now?
 379                          */
 380                         if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
 381                                 ASSERT3P(db->db_blkptr, ==,
 382                                     ((blkptr_t *)db->db_parent->db.db_data +
 383                                     db->db_blkid % epb));
 384                         }
 385                 }
 386         }
 387         if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
 388             (db->db_buf == NULL || db->db_buf->b_data) &&
 389             db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
 390             db->db_state != DB_FILL && !dn->dn_free_txg) {
 391                 /*
 392                  * If the blkptr isn't set but they have nonzero data,
 393                  * it had better be dirty, otherwise we'll lose that
 394                  * data when we evict this buffer.
 395                  */
 396                 if (db->db_dirtycnt == 0) {
 397                         uint64_t *buf = db->db.db_data;
 398                         int i;
 399 
 400                         for (i = 0; i < db->db.db_size >> 3; i++) {
 401                                 ASSERT(buf[i] == 0);
 402                         }
 403                 }
 404         }
 405         DB_DNODE_EXIT(db);
 406 }
 407 #endif
 408 
 409 static void
 410 dbuf_update_data(dmu_buf_impl_t *db)
 411 {
 412         ASSERT(MUTEX_HELD(&db->db_mtx));
 413         if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
 414                 ASSERT(!refcount_is_zero(&db->db_holds));
 415                 *db->db_user_data_ptr_ptr = db->db.db_data;
 416         }
 417 }
 418 
 419 static void
 420 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 421 {
 422         ASSERT(MUTEX_HELD(&db->db_mtx));
 423         ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
 424         db->db_buf = buf;
 425         if (buf != NULL) {
 426                 ASSERT(buf->b_data != NULL);
 427                 db->db.db_data = buf->b_data;
 428                 if (!arc_released(buf))
 429                         arc_set_callback(buf, dbuf_do_evict, db);
 430                 dbuf_update_data(db);
 431         } else {
 432                 dbuf_evict_user(db);
 433                 db->db.db_data = NULL;
 434                 if (db->db_state != DB_NOFILL)
 435                         db->db_state = DB_UNCACHED;
 436         }
 437 }
 438 
 439 /*
 440  * Loan out an arc_buf for read.  Return the loaned arc_buf.
 441  */
 442 arc_buf_t *
 443 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
 444 {
 445         arc_buf_t *abuf;
 446 
 447         mutex_enter(&db->db_mtx);
 448         if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
 449                 int blksz = db->db.db_size;
 450                 spa_t *spa;
 451 
 452                 mutex_exit(&db->db_mtx);
 453                 DB_GET_SPA(&spa, db);
 454                 abuf = arc_loan_buf(spa, blksz);
 455                 bcopy(db->db.db_data, abuf->b_data, blksz);
 456         } else {
 457                 abuf = db->db_buf;
 458                 arc_loan_inuse_buf(abuf, db);
 459                 dbuf_set_data(db, NULL);
 460                 mutex_exit(&db->db_mtx);
 461         }
 462         return (abuf);
 463 }
 464 
 465 uint64_t
 466 dbuf_whichblock(dnode_t *dn, uint64_t offset)
 467 {
 468         if (dn->dn_datablkshift) {
 469                 return (offset >> dn->dn_datablkshift);
 470         } else {
 471                 ASSERT3U(offset, <, dn->dn_datablksz);
 472                 return (0);
 473         }
 474 }
 475 
 476 static void
 477 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
 478 {
 479         dmu_buf_impl_t *db = vdb;
 480 
 481         mutex_enter(&db->db_mtx);
 482         ASSERT3U(db->db_state, ==, DB_READ);
 483         /*
 484          * All reads are synchronous, so we must have a hold on the dbuf
 485          */
 486         ASSERT(refcount_count(&db->db_holds) > 0);
 487         ASSERT(db->db_buf == NULL);
 488         ASSERT(db->db.db_data == NULL);
 489         if (db->db_level == 0 && db->db_freed_in_flight) {
 490                 /* we were freed in flight; disregard any error */
 491                 arc_release(buf, db);
 492                 bzero(buf->b_data, db->db.db_size);
 493                 arc_buf_freeze(buf);
 494                 db->db_freed_in_flight = FALSE;
 495                 dbuf_set_data(db, buf);
 496                 db->db_state = DB_CACHED;
 497         } else if (zio == NULL || zio->io_error == 0) {
 498                 dbuf_set_data(db, buf);
 499                 db->db_state = DB_CACHED;
 500         } else {
 501                 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 502                 ASSERT3P(db->db_buf, ==, NULL);
 503                 VERIFY(arc_buf_remove_ref(buf, db) == 1);
 504                 db->db_state = DB_UNCACHED;
 505         }
 506         cv_broadcast(&db->db_changed);
 507         dbuf_rele_and_unlock(db, NULL);
 508 }
 509 
 510 static void
 511 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
 512 {
 513         dnode_t *dn;
 514         spa_t *spa;
 515         zbookmark_t zb;
 516         uint32_t aflags = ARC_NOWAIT;
 517         arc_buf_t *pbuf;
 518 
 519         DB_DNODE_ENTER(db);
 520         dn = DB_DNODE(db);
 521         ASSERT(!refcount_is_zero(&db->db_holds));
 522         /* We need the struct_rwlock to prevent db_blkptr from changing. */
 523         ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
 524         ASSERT(MUTEX_HELD(&db->db_mtx));
 525         ASSERT(db->db_state == DB_UNCACHED);
 526         ASSERT(db->db_buf == NULL);
 527 
 528         if (db->db_blkid == DMU_BONUS_BLKID) {
 529                 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
 530 
 531                 ASSERT3U(bonuslen, <=, db->db.db_size);
 532                 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
 533                 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
 534                 if (bonuslen < DN_MAX_BONUSLEN)
 535                         bzero(db->db.db_data, DN_MAX_BONUSLEN);
 536 
 537                 if (bonuslen) {
 538                         /*
 539                          * Absent byzantine on-disk corruption, we fully expect
 540                          * our bonuslen to be no more than DN_MAX_BONUSLEN --
 541                          * but we nonetheless explicitly clamp it on the bcopy()
 542                          * to prevent any on-disk corruption from becoming
 543                          * rampant in-kernel corruption.
 544                          */
 545                         bcopy(DN_BONUS(dn->dn_phys), db->db.db_data,
 546                             MIN(bonuslen, DN_MAX_BONUSLEN));
 547                 }
 548 
 549                 DB_DNODE_EXIT(db);
 550                 dbuf_update_data(db);
 551                 db->db_state = DB_CACHED;
 552                 mutex_exit(&db->db_mtx);
 553                 return;
 554         }
 555 
 556         /*
 557          * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
 558          * processes the delete record and clears the bp while we are waiting
 559          * for the dn_mtx (resulting in a "no" from block_freed).
 560          */
 561         if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
 562             (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
 563             BP_IS_HOLE(db->db_blkptr)))) {
 564                 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
 565 
 566                 dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa,
 567                     db->db.db_size, db, type));
 568                 DB_DNODE_EXIT(db);
 569                 bzero(db->db.db_data, db->db.db_size);
 570                 db->db_state = DB_CACHED;
 571                 *flags |= DB_RF_CACHED;
 572                 mutex_exit(&db->db_mtx);
 573                 return;
 574         }
 575 
 576         spa = dn->dn_objset->os_spa;
 577         DB_DNODE_EXIT(db);
 578 
 579         db->db_state = DB_READ;
 580         mutex_exit(&db->db_mtx);
 581 
 582         if (DBUF_IS_L2CACHEABLE(db))
 583                 aflags |= ARC_L2CACHE;
 584 
 585         SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
 586             db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
 587             db->db.db_object, db->db_level, db->db_blkid);
 588 
 589         dbuf_add_ref(db, NULL);
 590         /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */
 591 
 592         if (db->db_parent)
 593                 pbuf = db->db_parent->db_buf;
 594         else
 595                 pbuf = db->db_objset->os_phys_buf;
 596 
 597         (void) dsl_read(zio, spa, db->db_blkptr, pbuf,
 598             dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
 599             (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
 600             &aflags, &zb);
 601         if (aflags & ARC_CACHED)
 602                 *flags |= DB_RF_CACHED;
 603 }
 604 
 605 int
 606 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
 607 {
 608         int err = 0;
 609         int havepzio = (zio != NULL);
 610         int prefetch;
 611         dnode_t *dn;
 612 
 613         /*
 614          * We don't have to hold the mutex to check db_state because it
 615          * can't be freed while we have a hold on the buffer.
 616          */
 617         ASSERT(!refcount_is_zero(&db->db_holds));
 618 
 619         if (db->db_state == DB_NOFILL)
 620                 return (EIO);
 621 
 622         DB_DNODE_ENTER(db);
 623         dn = DB_DNODE(db);
 624         if ((flags & DB_RF_HAVESTRUCT) == 0)
 625                 rw_enter(&dn->dn_struct_rwlock, RW_READER);
 626 
 627         prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
 628             (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL &&
 629             DBUF_IS_CACHEABLE(db);
 630 
 631         mutex_enter(&db->db_mtx);
 632         if (db->db_state == DB_CACHED) {
 633                 mutex_exit(&db->db_mtx);
 634                 if (prefetch)
 635                         dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
 636                             db->db.db_size, TRUE);
 637                 if ((flags & DB_RF_HAVESTRUCT) == 0)
 638                         rw_exit(&dn->dn_struct_rwlock);
 639                 DB_DNODE_EXIT(db);
 640         } else if (db->db_state == DB_UNCACHED) {
 641                 spa_t *spa = dn->dn_objset->os_spa;
 642 
 643                 if (zio == NULL)
 644                         zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
 645                 dbuf_read_impl(db, zio, &flags);
 646 
 647                 /* dbuf_read_impl has dropped db_mtx for us */
 648 
 649                 if (prefetch)
 650                         dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
 651                             db->db.db_size, flags & DB_RF_CACHED);
 652 
 653                 if ((flags & DB_RF_HAVESTRUCT) == 0)
 654                         rw_exit(&dn->dn_struct_rwlock);
 655                 DB_DNODE_EXIT(db);
 656 
 657                 if (!havepzio)
 658                         err = zio_wait(zio);
 659         } else {
 660                 mutex_exit(&db->db_mtx);
 661                 if (prefetch)
 662                         dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
 663                             db->db.db_size, TRUE);
 664                 if ((flags & DB_RF_HAVESTRUCT) == 0)
 665                         rw_exit(&dn->dn_struct_rwlock);
 666                 DB_DNODE_EXIT(db);
 667 
 668                 mutex_enter(&db->db_mtx);
 669                 if ((flags & DB_RF_NEVERWAIT) == 0) {
 670                         while (db->db_state == DB_READ ||
 671                             db->db_state == DB_FILL) {
 672                                 ASSERT(db->db_state == DB_READ ||
 673                                     (flags & DB_RF_HAVESTRUCT) == 0);
 674                                 cv_wait(&db->db_changed, &db->db_mtx);
 675                         }
 676                         if (db->db_state == DB_UNCACHED)
 677                                 err = EIO;
 678                 }
 679                 mutex_exit(&db->db_mtx);
 680         }
 681 
 682         ASSERT(err || havepzio || db->db_state == DB_CACHED);
 683         return (err);
 684 }
 685 
 686 static void
 687 dbuf_noread(dmu_buf_impl_t *db)
 688 {
 689         ASSERT(!refcount_is_zero(&db->db_holds));
 690         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 691         mutex_enter(&db->db_mtx);
 692         while (db->db_state == DB_READ || db->db_state == DB_FILL)
 693                 cv_wait(&db->db_changed, &db->db_mtx);
 694         if (db->db_state == DB_UNCACHED) {
 695                 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
 696                 spa_t *spa;
 697 
 698                 ASSERT(db->db_buf == NULL);
 699                 ASSERT(db->db.db_data == NULL);
 700                 DB_GET_SPA(&spa, db);
 701                 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
 702                 db->db_state = DB_FILL;
 703         } else if (db->db_state == DB_NOFILL) {
 704                 dbuf_set_data(db, NULL);
 705         } else {
 706                 ASSERT3U(db->db_state, ==, DB_CACHED);
 707         }
 708         mutex_exit(&db->db_mtx);
 709 }
 710 
 711 /*
 712  * This is our just-in-time copy function.  It makes a copy of
 713  * buffers, that have been modified in a previous transaction
 714  * group, before we modify them in the current active group.
 715  *
 716  * This function is used in two places: when we are dirtying a
 717  * buffer for the first time in a txg, and when we are freeing
 718  * a range in a dnode that includes this buffer.
 719  *
 720  * Note that when we are called from dbuf_free_range() we do
 721  * not put a hold on the buffer, we just traverse the active
 722  * dbuf list for the dnode.
 723  */
 724 static void
 725 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
 726 {
 727         dbuf_dirty_record_t *dr = db->db_last_dirty;
 728 
 729         ASSERT(MUTEX_HELD(&db->db_mtx));
 730         ASSERT(db->db.db_data != NULL);
 731         ASSERT(db->db_level == 0);
 732         ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
 733 
 734         if (dr == NULL ||
 735             (dr->dt.dl.dr_data !=
 736             ((db->db_blkid  == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
 737                 return;
 738 
 739         /*
 740          * If the last dirty record for this dbuf has not yet synced
 741          * and its referencing the dbuf data, either:
 742          *      reset the reference to point to a new copy,
 743          * or (if there a no active holders)
 744          *      just null out the current db_data pointer.
 745          */
 746         ASSERT(dr->dr_txg >= txg - 2);
 747         if (db->db_blkid == DMU_BONUS_BLKID) {
 748                 /* Note that the data bufs here are zio_bufs */
 749                 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
 750                 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
 751                 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
 752         } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
 753                 int size = db->db.db_size;
 754                 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
 755                 spa_t *spa;
 756 
 757                 DB_GET_SPA(&spa, db);
 758                 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
 759                 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
 760         } else {
 761                 dbuf_set_data(db, NULL);
 762         }
 763 }
 764 
 765 void
 766 dbuf_unoverride(dbuf_dirty_record_t *dr)
 767 {
 768         dmu_buf_impl_t *db = dr->dr_dbuf;
 769         blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
 770         uint64_t txg = dr->dr_txg;
 771 
 772         ASSERT(MUTEX_HELD(&db->db_mtx));
 773         ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
 774         ASSERT(db->db_level == 0);
 775 
 776         if (db->db_blkid == DMU_BONUS_BLKID ||
 777             dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
 778                 return;
 779 
 780         ASSERT(db->db_data_pending != dr);
 781 
 782         /* free this block */
 783         if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) {
 784                 spa_t *spa;
 785 
 786                 DB_GET_SPA(&spa, db);
 787                 zio_free(spa, txg, bp);
 788         }
 789         dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
 790         dr->dt.dl.dr_nopwrite = B_FALSE;
 791 
 792         /*
 793          * Release the already-written buffer, so we leave it in
 794          * a consistent dirty state.  Note that all callers are
 795          * modifying the buffer, so they will immediately do
 796          * another (redundant) arc_release().  Therefore, leave
 797          * the buf thawed to save the effort of freezing &
 798          * immediately re-thawing it.
 799          */
 800         arc_release(dr->dt.dl.dr_data, db);
 801 }
 802 
 803 /*
 804  * Evict (if its unreferenced) or clear (if its referenced) any level-0
 805  * data blocks in the free range, so that any future readers will find
 806  * empty blocks.  Also, if we happen accross any level-1 dbufs in the
 807  * range that have not already been marked dirty, mark them dirty so
 808  * they stay in memory.
 809  */
 810 void
 811 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
 812 {
 813         dmu_buf_impl_t *db, *db_next;
 814         uint64_t txg = tx->tx_txg;
 815         int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
 816         uint64_t first_l1 = start >> epbs;
 817         uint64_t last_l1 = end >> epbs;
 818 
 819         if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) {
 820                 end = dn->dn_maxblkid;
 821                 last_l1 = end >> epbs;
 822         }
 823         dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
 824         mutex_enter(&dn->dn_dbufs_mtx);
 825         for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
 826                 db_next = list_next(&dn->dn_dbufs, db);
 827                 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 828 
 829                 if (db->db_level == 1 &&
 830                     db->db_blkid >= first_l1 && db->db_blkid <= last_l1) {
 831                         mutex_enter(&db->db_mtx);
 832                         if (db->db_last_dirty &&
 833                             db->db_last_dirty->dr_txg < txg) {
 834                                 dbuf_add_ref(db, FTAG);
 835                                 mutex_exit(&db->db_mtx);
 836                                 dbuf_will_dirty(db, tx);
 837                                 dbuf_rele(db, FTAG);
 838                         } else {
 839                                 mutex_exit(&db->db_mtx);
 840                         }
 841                 }
 842 
 843                 if (db->db_level != 0)
 844                         continue;
 845                 dprintf_dbuf(db, "found buf %s\n", "");
 846                 if (db->db_blkid < start || db->db_blkid > end)
 847                         continue;
 848 
 849                 /* found a level 0 buffer in the range */
 850                 if (dbuf_undirty(db, tx))
 851                         continue;
 852 
 853                 mutex_enter(&db->db_mtx);
 854                 if (db->db_state == DB_UNCACHED ||
 855                     db->db_state == DB_NOFILL ||
 856                     db->db_state == DB_EVICTING) {
 857                         ASSERT(db->db.db_data == NULL);
 858                         mutex_exit(&db->db_mtx);
 859                         continue;
 860                 }
 861                 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
 862                         /* will be handled in dbuf_read_done or dbuf_rele */
 863                         db->db_freed_in_flight = TRUE;
 864                         mutex_exit(&db->db_mtx);
 865                         continue;
 866                 }
 867                 if (refcount_count(&db->db_holds) == 0) {
 868                         ASSERT(db->db_buf);
 869                         dbuf_clear(db);
 870                         continue;
 871                 }
 872                 /* The dbuf is referenced */
 873 
 874                 if (db->db_last_dirty != NULL) {
 875                         dbuf_dirty_record_t *dr = db->db_last_dirty;
 876 
 877                         if (dr->dr_txg == txg) {
 878                                 /*
 879                                  * This buffer is "in-use", re-adjust the file
 880                                  * size to reflect that this buffer may
 881                                  * contain new data when we sync.
 882                                  */
 883                                 if (db->db_blkid != DMU_SPILL_BLKID &&
 884                                     db->db_blkid > dn->dn_maxblkid)
 885                                         dn->dn_maxblkid = db->db_blkid;
 886                                 dbuf_unoverride(dr);
 887                         } else {
 888                                 /*
 889                                  * This dbuf is not dirty in the open context.
 890                                  * Either uncache it (if its not referenced in
 891                                  * the open context) or reset its contents to
 892                                  * empty.
 893                                  */
 894                                 dbuf_fix_old_data(db, txg);
 895                         }
 896                 }
 897                 /* clear the contents if its cached */
 898                 if (db->db_state == DB_CACHED) {
 899                         ASSERT(db->db.db_data != NULL);
 900                         arc_release(db->db_buf, db);
 901                         bzero(db->db.db_data, db->db.db_size);
 902                         arc_buf_freeze(db->db_buf);
 903                 }
 904 
 905                 mutex_exit(&db->db_mtx);
 906         }
 907         mutex_exit(&dn->dn_dbufs_mtx);
 908 }
 909 
 910 static int
 911 dbuf_block_freeable(dmu_buf_impl_t *db)
 912 {
 913         dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
 914         uint64_t birth_txg = 0;
 915 
 916         /*
 917          * We don't need any locking to protect db_blkptr:
 918          * If it's syncing, then db_last_dirty will be set
 919          * so we'll ignore db_blkptr.
 920          */
 921         ASSERT(MUTEX_HELD(&db->db_mtx));
 922         if (db->db_last_dirty)
 923                 birth_txg = db->db_last_dirty->dr_txg;
 924         else if (db->db_blkptr)
 925                 birth_txg = db->db_blkptr->blk_birth;
 926 
 927         /*
 928          * If we don't exist or are in a snapshot, we can't be freed.
 929          * Don't pass the bp to dsl_dataset_block_freeable() since we
 930          * are holding the db_mtx lock and might deadlock if we are
 931          * prefetching a dedup-ed block.
 932          */
 933         if (birth_txg)
 934                 return (ds == NULL ||
 935                     dsl_dataset_block_freeable(ds, NULL, birth_txg));
 936         else
 937                 return (FALSE);
 938 }
 939 
 940 void
 941 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
 942 {
 943         arc_buf_t *buf, *obuf;
 944         int osize = db->db.db_size;
 945         arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
 946         dnode_t *dn;
 947 
 948         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
 949 
 950         DB_DNODE_ENTER(db);
 951         dn = DB_DNODE(db);
 952 
 953         /* XXX does *this* func really need the lock? */
 954         ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
 955 
 956         /*
 957          * This call to dbuf_will_dirty() with the dn_struct_rwlock held
 958          * is OK, because there can be no other references to the db
 959          * when we are changing its size, so no concurrent DB_FILL can
 960          * be happening.
 961          */
 962         /*
 963          * XXX we should be doing a dbuf_read, checking the return
 964          * value and returning that up to our callers
 965          */
 966         dbuf_will_dirty(db, tx);
 967 
 968         /* create the data buffer for the new block */
 969         buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
 970 
 971         /* copy old block data to the new block */
 972         obuf = db->db_buf;
 973         bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
 974         /* zero the remainder */
 975         if (size > osize)
 976                 bzero((uint8_t *)buf->b_data + osize, size - osize);
 977 
 978         mutex_enter(&db->db_mtx);
 979         dbuf_set_data(db, buf);
 980         VERIFY(arc_buf_remove_ref(obuf, db) == 1);
 981         db->db.db_size = size;
 982 
 983         if (db->db_level == 0) {
 984                 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
 985                 db->db_last_dirty->dt.dl.dr_data = buf;
 986         }
 987         mutex_exit(&db->db_mtx);
 988 
 989         dnode_willuse_space(dn, size-osize, tx);
 990         DB_DNODE_EXIT(db);
 991 }
 992 
 993 void
 994 dbuf_release_bp(dmu_buf_impl_t *db)
 995 {
 996         objset_t *os;
 997         zbookmark_t zb;
 998 
 999         DB_GET_OBJSET(&os, db);
1000         ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
1001         ASSERT(arc_released(os->os_phys_buf) ||
1002             list_link_active(&os->os_dsl_dataset->ds_synced_link));
1003         ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1004 
1005         zb.zb_objset = os->os_dsl_dataset ?
1006             os->os_dsl_dataset->ds_object : 0;
1007         zb.zb_object = db->db.db_object;
1008         zb.zb_level = db->db_level;
1009         zb.zb_blkid = db->db_blkid;
1010         (void) arc_release_bp(db->db_buf, db,
1011             db->db_blkptr, os->os_spa, &zb);
1012 }
1013 
1014 dbuf_dirty_record_t *
1015 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1016 {
1017         dnode_t *dn;
1018         objset_t *os;
1019         dbuf_dirty_record_t **drp, *dr;
1020         int drop_struct_lock = FALSE;
1021         boolean_t do_free_accounting = B_FALSE;
1022         int txgoff = tx->tx_txg & TXG_MASK;
1023 
1024         ASSERT(tx->tx_txg != 0);
1025         ASSERT(!refcount_is_zero(&db->db_holds));
1026         DMU_TX_DIRTY_BUF(tx, db);
1027 
1028         DB_DNODE_ENTER(db);
1029         dn = DB_DNODE(db);
1030         /*
1031          * Shouldn't dirty a regular buffer in syncing context.  Private
1032          * objects may be dirtied in syncing context, but only if they
1033          * were already pre-dirtied in open context.
1034          */
1035         ASSERT(!dmu_tx_is_syncing(tx) ||
1036             BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
1037             DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1038             dn->dn_objset->os_dsl_dataset == NULL);
1039         /*
1040          * We make this assert for private objects as well, but after we
1041          * check if we're already dirty.  They are allowed to re-dirty
1042          * in syncing context.
1043          */
1044         ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1045             dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1046             (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1047 
1048         mutex_enter(&db->db_mtx);
1049         /*
1050          * XXX make this true for indirects too?  The problem is that
1051          * transactions created with dmu_tx_create_assigned() from
1052          * syncing context don't bother holding ahead.
1053          */
1054         ASSERT(db->db_level != 0 ||
1055             db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1056             db->db_state == DB_NOFILL);
1057 
1058         mutex_enter(&dn->dn_mtx);
1059         /*
1060          * Don't set dirtyctx to SYNC if we're just modifying this as we
1061          * initialize the objset.
1062          */
1063         if (dn->dn_dirtyctx == DN_UNDIRTIED &&
1064             !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
1065                 dn->dn_dirtyctx =
1066                     (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
1067                 ASSERT(dn->dn_dirtyctx_firstset == NULL);
1068                 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
1069         }
1070         mutex_exit(&dn->dn_mtx);
1071 
1072         if (db->db_blkid == DMU_SPILL_BLKID)
1073                 dn->dn_have_spill = B_TRUE;
1074 
1075         /*
1076          * If this buffer is already dirty, we're done.
1077          */
1078         drp = &db->db_last_dirty;
1079         ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
1080             db->db.db_object == DMU_META_DNODE_OBJECT);
1081         while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
1082                 drp = &dr->dr_next;
1083         if (dr && dr->dr_txg == tx->tx_txg) {
1084                 DB_DNODE_EXIT(db);
1085 
1086                 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1087                         /*
1088                          * If this buffer has already been written out,
1089                          * we now need to reset its state.
1090                          */
1091                         dbuf_unoverride(dr);
1092                         if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1093                             db->db_state != DB_NOFILL)
1094                                 arc_buf_thaw(db->db_buf);
1095                 }
1096                 mutex_exit(&db->db_mtx);
1097                 return (dr);
1098         }
1099 
1100         /*
1101          * Only valid if not already dirty.
1102          */
1103         ASSERT(dn->dn_object == 0 ||
1104             dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1105             (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1106 
1107         ASSERT3U(dn->dn_nlevels, >, db->db_level);
1108         ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1109             dn->dn_phys->dn_nlevels > db->db_level ||
1110             dn->dn_next_nlevels[txgoff] > db->db_level ||
1111             dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1112             dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1113 
1114         /*
1115          * We should only be dirtying in syncing context if it's the
1116          * mos or we're initializing the os or it's a special object.
1117          * However, we are allowed to dirty in syncing context provided
1118          * we already dirtied it in open context.  Hence we must make
1119          * this assertion only if we're not already dirty.
1120          */
1121         os = dn->dn_objset;
1122         ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1123             os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp));
1124         ASSERT(db->db.db_size != 0);
1125 
1126         dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1127 
1128         if (db->db_blkid != DMU_BONUS_BLKID) {
1129                 /*
1130                  * Update the accounting.
1131                  * Note: we delay "free accounting" until after we drop
1132                  * the db_mtx.  This keeps us from grabbing other locks
1133                  * (and possibly deadlocking) in bp_get_dsize() while
1134                  * also holding the db_mtx.
1135                  */
1136                 dnode_willuse_space(dn, db->db.db_size, tx);
1137                 do_free_accounting = dbuf_block_freeable(db);
1138         }
1139 
1140         /*
1141          * If this buffer is dirty in an old transaction group we need
1142          * to make a copy of it so that the changes we make in this
1143          * transaction group won't leak out when we sync the older txg.
1144          */
1145         dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1146         if (db->db_level == 0) {
1147                 void *data_old = db->db_buf;
1148 
1149                 if (db->db_state != DB_NOFILL) {
1150                         if (db->db_blkid == DMU_BONUS_BLKID) {
1151                                 dbuf_fix_old_data(db, tx->tx_txg);
1152                                 data_old = db->db.db_data;
1153                         } else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1154                                 /*
1155                                  * Release the data buffer from the cache so
1156                                  * that we can modify it without impacting
1157                                  * possible other users of this cached data
1158                                  * block.  Note that indirect blocks and
1159                                  * private objects are not released until the
1160                                  * syncing state (since they are only modified
1161                                  * then).
1162                                  */
1163                                 arc_release(db->db_buf, db);
1164                                 dbuf_fix_old_data(db, tx->tx_txg);
1165                                 data_old = db->db_buf;
1166                         }
1167                         ASSERT(data_old != NULL);
1168                 }
1169                 dr->dt.dl.dr_data = data_old;
1170         } else {
1171                 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1172                 list_create(&dr->dt.di.dr_children,
1173                     sizeof (dbuf_dirty_record_t),
1174                     offsetof(dbuf_dirty_record_t, dr_dirty_node));
1175         }
1176         dr->dr_dbuf = db;
1177         dr->dr_txg = tx->tx_txg;
1178         dr->dr_next = *drp;
1179         *drp = dr;
1180 
1181         /*
1182          * We could have been freed_in_flight between the dbuf_noread
1183          * and dbuf_dirty.  We win, as though the dbuf_noread() had
1184          * happened after the free.
1185          */
1186         if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1187             db->db_blkid != DMU_SPILL_BLKID) {
1188                 mutex_enter(&dn->dn_mtx);
1189                 dnode_clear_range(dn, db->db_blkid, 1, tx);
1190                 mutex_exit(&dn->dn_mtx);
1191                 db->db_freed_in_flight = FALSE;
1192         }
1193 
1194         /*
1195          * This buffer is now part of this txg
1196          */
1197         dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1198         db->db_dirtycnt += 1;
1199         ASSERT3U(db->db_dirtycnt, <=, 3);
1200 
1201         mutex_exit(&db->db_mtx);
1202 
1203         if (db->db_blkid == DMU_BONUS_BLKID ||
1204             db->db_blkid == DMU_SPILL_BLKID) {
1205                 mutex_enter(&dn->dn_mtx);
1206                 ASSERT(!list_link_active(&dr->dr_dirty_node));
1207                 list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1208                 mutex_exit(&dn->dn_mtx);
1209                 dnode_setdirty(dn, tx);
1210                 DB_DNODE_EXIT(db);
1211                 return (dr);
1212         } else if (do_free_accounting) {
1213                 blkptr_t *bp = db->db_blkptr;
1214                 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1215                     bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1216                 /*
1217                  * This is only a guess -- if the dbuf is dirty
1218                  * in a previous txg, we don't know how much
1219                  * space it will use on disk yet.  We should
1220                  * really have the struct_rwlock to access
1221                  * db_blkptr, but since this is just a guess,
1222                  * it's OK if we get an odd answer.
1223                  */
1224                 ddt_prefetch(os->os_spa, bp);
1225                 dnode_willuse_space(dn, -willfree, tx);
1226         }
1227 
1228         if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1229                 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1230                 drop_struct_lock = TRUE;
1231         }
1232 
1233         if (db->db_level == 0) {
1234                 dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1235                 ASSERT(dn->dn_maxblkid >= db->db_blkid);
1236         }
1237 
1238         if (db->db_level+1 < dn->dn_nlevels) {
1239                 dmu_buf_impl_t *parent = db->db_parent;
1240                 dbuf_dirty_record_t *di;
1241                 int parent_held = FALSE;
1242 
1243                 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1244                         int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1245 
1246                         parent = dbuf_hold_level(dn, db->db_level+1,
1247                             db->db_blkid >> epbs, FTAG);
1248                         ASSERT(parent != NULL);
1249                         parent_held = TRUE;
1250                 }
1251                 if (drop_struct_lock)
1252                         rw_exit(&dn->dn_struct_rwlock);
1253                 ASSERT3U(db->db_level+1, ==, parent->db_level);
1254                 di = dbuf_dirty(parent, tx);
1255                 if (parent_held)
1256                         dbuf_rele(parent, FTAG);
1257 
1258                 mutex_enter(&db->db_mtx);
1259                 /*  possible race with dbuf_undirty() */
1260                 if (db->db_last_dirty == dr ||
1261                     dn->dn_object == DMU_META_DNODE_OBJECT) {
1262                         mutex_enter(&di->dt.di.dr_mtx);
1263                         ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1264                         ASSERT(!list_link_active(&dr->dr_dirty_node));
1265                         list_insert_tail(&di->dt.di.dr_children, dr);
1266                         mutex_exit(&di->dt.di.dr_mtx);
1267                         dr->dr_parent = di;
1268                 }
1269                 mutex_exit(&db->db_mtx);
1270         } else {
1271                 ASSERT(db->db_level+1 == dn->dn_nlevels);
1272                 ASSERT(db->db_blkid < dn->dn_nblkptr);
1273                 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1274                 mutex_enter(&dn->dn_mtx);
1275                 ASSERT(!list_link_active(&dr->dr_dirty_node));
1276                 list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1277                 mutex_exit(&dn->dn_mtx);
1278                 if (drop_struct_lock)
1279                         rw_exit(&dn->dn_struct_rwlock);
1280         }
1281 
1282         dnode_setdirty(dn, tx);
1283         DB_DNODE_EXIT(db);
1284         return (dr);
1285 }
1286 
1287 static int
1288 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1289 {
1290         dnode_t *dn;
1291         uint64_t txg = tx->tx_txg;
1292         dbuf_dirty_record_t *dr, **drp;
1293 
1294         ASSERT(txg != 0);
1295         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1296 
1297         mutex_enter(&db->db_mtx);
1298         /*
1299          * If this buffer is not dirty, we're done.
1300          */
1301         for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1302                 if (dr->dr_txg <= txg)
1303                         break;
1304         if (dr == NULL || dr->dr_txg < txg) {
1305                 mutex_exit(&db->db_mtx);
1306                 return (0);
1307         }
1308         ASSERT(dr->dr_txg == txg);
1309         ASSERT(dr->dr_dbuf == db);
1310 
1311         DB_DNODE_ENTER(db);
1312         dn = DB_DNODE(db);
1313 
1314         /*
1315          * If this buffer is currently held, we cannot undirty
1316          * it, since one of the current holders may be in the
1317          * middle of an update.  Note that users of dbuf_undirty()
1318          * should not place a hold on the dbuf before the call.
1319          * Also note: we can get here with a spill block, so
1320          * test for that similar to how dbuf_dirty does.
1321          */
1322         if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1323                 mutex_exit(&db->db_mtx);
1324                 /* Make sure we don't toss this buffer at sync phase */
1325                 if (db->db_blkid != DMU_SPILL_BLKID) {
1326                         mutex_enter(&dn->dn_mtx);
1327                         dnode_clear_range(dn, db->db_blkid, 1, tx);
1328                         mutex_exit(&dn->dn_mtx);
1329                 }
1330                 DB_DNODE_EXIT(db);
1331                 return (0);
1332         }
1333 
1334         dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1335 
1336         ASSERT(db->db.db_size != 0);
1337 
1338         /* XXX would be nice to fix up dn_towrite_space[] */
1339 
1340         *drp = dr->dr_next;
1341 
1342         /*
1343          * Note that there are three places in dbuf_dirty()
1344          * where this dirty record may be put on a list.
1345          * Make sure to do a list_remove corresponding to
1346          * every one of those list_insert calls.
1347          */
1348         if (dr->dr_parent) {
1349                 mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1350                 list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1351                 mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1352         } else if (db->db_blkid == DMU_SPILL_BLKID ||
1353             db->db_level+1 == dn->dn_nlevels) {
1354                 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1355                 mutex_enter(&dn->dn_mtx);
1356                 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1357                 mutex_exit(&dn->dn_mtx);
1358         }
1359         DB_DNODE_EXIT(db);
1360 
1361         if (db->db_level == 0) {
1362                 if (db->db_state != DB_NOFILL) {
1363                         dbuf_unoverride(dr);
1364 
1365                         ASSERT(db->db_buf != NULL);
1366                         ASSERT(dr->dt.dl.dr_data != NULL);
1367                         if (dr->dt.dl.dr_data != db->db_buf)
1368                                 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
1369                                     db) == 1);
1370                 }
1371         } else {
1372                 ASSERT(db->db_buf != NULL);
1373                 ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1374                 mutex_destroy(&dr->dt.di.dr_mtx);
1375                 list_destroy(&dr->dt.di.dr_children);
1376         }
1377         kmem_free(dr, sizeof (dbuf_dirty_record_t));
1378 
1379         ASSERT(db->db_dirtycnt > 0);
1380         db->db_dirtycnt -= 1;
1381 
1382         if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1383                 arc_buf_t *buf = db->db_buf;
1384 
1385                 ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1386                 dbuf_set_data(db, NULL);
1387                 VERIFY(arc_buf_remove_ref(buf, db) == 1);
1388                 dbuf_evict(db);
1389                 return (1);
1390         }
1391 
1392         mutex_exit(&db->db_mtx);
1393         return (0);
1394 }
1395 
1396 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1397 void
1398 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1399 {
1400         int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1401 
1402         ASSERT(tx->tx_txg != 0);
1403         ASSERT(!refcount_is_zero(&db->db_holds));
1404 
1405         DB_DNODE_ENTER(db);
1406         if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1407                 rf |= DB_RF_HAVESTRUCT;
1408         DB_DNODE_EXIT(db);
1409         (void) dbuf_read(db, NULL, rf);
1410         (void) dbuf_dirty(db, tx);
1411 }
1412 
1413 void
1414 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1415 {
1416         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1417 
1418         db->db_state = DB_NOFILL;
1419 
1420         dmu_buf_will_fill(db_fake, tx);
1421 }
1422 
1423 void
1424 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1425 {
1426         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1427 
1428         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1429         ASSERT(tx->tx_txg != 0);
1430         ASSERT(db->db_level == 0);
1431         ASSERT(!refcount_is_zero(&db->db_holds));
1432 
1433         ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1434             dmu_tx_private_ok(tx));
1435 
1436         dbuf_noread(db);
1437         (void) dbuf_dirty(db, tx);
1438 }
1439 
1440 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1441 /* ARGSUSED */
1442 void
1443 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1444 {
1445         mutex_enter(&db->db_mtx);
1446         DBUF_VERIFY(db);
1447 
1448         if (db->db_state == DB_FILL) {
1449                 if (db->db_level == 0 && db->db_freed_in_flight) {
1450                         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1451                         /* we were freed while filling */
1452                         /* XXX dbuf_undirty? */
1453                         bzero(db->db.db_data, db->db.db_size);
1454                         db->db_freed_in_flight = FALSE;
1455                 }
1456                 db->db_state = DB_CACHED;
1457                 cv_broadcast(&db->db_changed);
1458         }
1459         mutex_exit(&db->db_mtx);
1460 }
1461 
1462 /*
1463  * Directly assign a provided arc buf to a given dbuf if it's not referenced
1464  * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
1465  */
1466 void
1467 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1468 {
1469         ASSERT(!refcount_is_zero(&db->db_holds));
1470         ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1471         ASSERT(db->db_level == 0);
1472         ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1473         ASSERT(buf != NULL);
1474         ASSERT(arc_buf_size(buf) == db->db.db_size);
1475         ASSERT(tx->tx_txg != 0);
1476 
1477         arc_return_buf(buf, db);
1478         ASSERT(arc_released(buf));
1479 
1480         mutex_enter(&db->db_mtx);
1481 
1482         while (db->db_state == DB_READ || db->db_state == DB_FILL)
1483                 cv_wait(&db->db_changed, &db->db_mtx);
1484 
1485         ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1486 
1487         if (db->db_state == DB_CACHED &&
1488             refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1489                 mutex_exit(&db->db_mtx);
1490                 (void) dbuf_dirty(db, tx);
1491                 bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1492                 VERIFY(arc_buf_remove_ref(buf, db) == 1);
1493                 xuio_stat_wbuf_copied();
1494                 return;
1495         }
1496 
1497         xuio_stat_wbuf_nocopy();
1498         if (db->db_state == DB_CACHED) {
1499                 dbuf_dirty_record_t *dr = db->db_last_dirty;
1500 
1501                 ASSERT(db->db_buf != NULL);
1502                 if (dr != NULL && dr->dr_txg == tx->tx_txg) {
1503                         ASSERT(dr->dt.dl.dr_data == db->db_buf);
1504                         if (!arc_released(db->db_buf)) {
1505                                 ASSERT(dr->dt.dl.dr_override_state ==
1506                                     DR_OVERRIDDEN);
1507                                 arc_release(db->db_buf, db);
1508                         }
1509                         dr->dt.dl.dr_data = buf;
1510                         VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1);
1511                 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1512                         arc_release(db->db_buf, db);
1513                         VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1);
1514                 }
1515                 db->db_buf = NULL;
1516         }
1517         ASSERT(db->db_buf == NULL);
1518         dbuf_set_data(db, buf);
1519         db->db_state = DB_FILL;
1520         mutex_exit(&db->db_mtx);
1521         (void) dbuf_dirty(db, tx);
1522         dbuf_fill_done(db, tx);
1523 }
1524 
1525 /*
1526  * "Clear" the contents of this dbuf.  This will mark the dbuf
1527  * EVICTING and clear *most* of its references.  Unfortunetely,
1528  * when we are not holding the dn_dbufs_mtx, we can't clear the
1529  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1530  * in this case.  For callers from the DMU we will usually see:
1531  *      dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1532  * For the arc callback, we will usually see:
1533  *      dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1534  * Sometimes, though, we will get a mix of these two:
1535  *      DMU: dbuf_clear()->arc_buf_evict()
1536  *      ARC: dbuf_do_evict()->dbuf_destroy()
1537  */
1538 void
1539 dbuf_clear(dmu_buf_impl_t *db)
1540 {
1541         dnode_t *dn;
1542         dmu_buf_impl_t *parent = db->db_parent;
1543         dmu_buf_impl_t *dndb;
1544         int dbuf_gone = FALSE;
1545 
1546         ASSERT(MUTEX_HELD(&db->db_mtx));
1547         ASSERT(refcount_is_zero(&db->db_holds));
1548 
1549         dbuf_evict_user(db);
1550 
1551         if (db->db_state == DB_CACHED) {
1552                 ASSERT(db->db.db_data != NULL);
1553                 if (db->db_blkid == DMU_BONUS_BLKID) {
1554                         zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1555                         arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
1556                 }
1557                 db->db.db_data = NULL;
1558                 db->db_state = DB_UNCACHED;
1559         }
1560 
1561         ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1562         ASSERT(db->db_data_pending == NULL);
1563 
1564         db->db_state = DB_EVICTING;
1565         db->db_blkptr = NULL;
1566 
1567         DB_DNODE_ENTER(db);
1568         dn = DB_DNODE(db);
1569         dndb = dn->dn_dbuf;
1570         if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1571                 list_remove(&dn->dn_dbufs, db);
1572                 (void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1573                 membar_producer();
1574                 DB_DNODE_EXIT(db);
1575                 /*
1576                  * Decrementing the dbuf count means that the hold corresponding
1577                  * to the removed dbuf is no longer discounted in dnode_move(),
1578                  * so the dnode cannot be moved until after we release the hold.
1579                  * The membar_producer() ensures visibility of the decremented
1580                  * value in dnode_move(), since DB_DNODE_EXIT doesn't actually
1581                  * release any lock.
1582                  */
1583                 dnode_rele(dn, db);
1584                 db->db_dnode_handle = NULL;
1585         } else {
1586                 DB_DNODE_EXIT(db);
1587         }
1588 
1589         if (db->db_buf)
1590                 dbuf_gone = arc_buf_evict(db->db_buf);
1591 
1592         if (!dbuf_gone)
1593                 mutex_exit(&db->db_mtx);
1594 
1595         /*
1596          * If this dbuf is referenced from an indirect dbuf,
1597          * decrement the ref count on the indirect dbuf.
1598          */
1599         if (parent && parent != dndb)
1600                 dbuf_rele(parent, db);
1601 }
1602 
1603 static int
1604 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1605     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1606 {
1607         int nlevels, epbs;
1608 
1609         *parentp = NULL;
1610         *bpp = NULL;
1611 
1612         ASSERT(blkid != DMU_BONUS_BLKID);
1613 
1614         if (blkid == DMU_SPILL_BLKID) {
1615                 mutex_enter(&dn->dn_mtx);
1616                 if (dn->dn_have_spill &&
1617                     (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
1618                         *bpp = &dn->dn_phys->dn_spill;
1619                 else
1620                         *bpp = NULL;
1621                 dbuf_add_ref(dn->dn_dbuf, NULL);
1622                 *parentp = dn->dn_dbuf;
1623                 mutex_exit(&dn->dn_mtx);
1624                 return (0);
1625         }
1626 
1627         if (dn->dn_phys->dn_nlevels == 0)
1628                 nlevels = 1;
1629         else
1630                 nlevels = dn->dn_phys->dn_nlevels;
1631 
1632         epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1633 
1634         ASSERT3U(level * epbs, <, 64);
1635         ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1636         if (level >= nlevels ||
1637             (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1638                 /* the buffer has no parent yet */
1639                 return (ENOENT);
1640         } else if (level < nlevels-1) {
1641                 /* this block is referenced from an indirect block */
1642                 int err = dbuf_hold_impl(dn, level+1,
1643                     blkid >> epbs, fail_sparse, NULL, parentp);
1644                 if (err)
1645                         return (err);
1646                 err = dbuf_read(*parentp, NULL,
1647                     (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1648                 if (err) {
1649                         dbuf_rele(*parentp, NULL);
1650                         *parentp = NULL;
1651                         return (err);
1652                 }
1653                 *bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1654                     (blkid & ((1ULL << epbs) - 1));
1655                 return (0);
1656         } else {
1657                 /* the block is referenced from the dnode */
1658                 ASSERT3U(level, ==, nlevels-1);
1659                 ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1660                     blkid < dn->dn_phys->dn_nblkptr);
1661                 if (dn->dn_dbuf) {
1662                         dbuf_add_ref(dn->dn_dbuf, NULL);
1663                         *parentp = dn->dn_dbuf;
1664                 }
1665                 *bpp = &dn->dn_phys->dn_blkptr[blkid];
1666                 return (0);
1667         }
1668 }
1669 
1670 static dmu_buf_impl_t *
1671 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1672     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1673 {
1674         objset_t *os = dn->dn_objset;
1675         dmu_buf_impl_t *db, *odb;
1676 
1677         ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1678         ASSERT(dn->dn_type != DMU_OT_NONE);
1679 
1680         db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1681 
1682         db->db_objset = os;
1683         db->db.db_object = dn->dn_object;
1684         db->db_level = level;
1685         db->db_blkid = blkid;
1686         db->db_last_dirty = NULL;
1687         db->db_dirtycnt = 0;
1688         db->db_dnode_handle = dn->dn_handle;
1689         db->db_parent = parent;
1690         db->db_blkptr = blkptr;
1691 
1692         db->db_user_ptr = NULL;
1693         db->db_user_data_ptr_ptr = NULL;
1694         db->db_evict_func = NULL;
1695         db->db_immediate_evict = 0;
1696         db->db_freed_in_flight = 0;
1697 
1698         if (blkid == DMU_BONUS_BLKID) {
1699                 ASSERT3P(parent, ==, dn->dn_dbuf);
1700                 db->db.db_size = DN_MAX_BONUSLEN -
1701                     (dn->dn_nblkptr-1) * sizeof (blkptr_t);
1702                 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1703                 db->db.db_offset = DMU_BONUS_BLKID;
1704                 db->db_state = DB_UNCACHED;
1705                 /* the bonus dbuf is not placed in the hash table */
1706                 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1707                 return (db);
1708         } else if (blkid == DMU_SPILL_BLKID) {
1709                 db->db.db_size = (blkptr != NULL) ?
1710                     BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
1711                 db->db.db_offset = 0;
1712         } else {
1713                 int blocksize =
1714                     db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1715                 db->db.db_size = blocksize;
1716                 db->db.db_offset = db->db_blkid * blocksize;
1717         }
1718 
1719         /*
1720          * Hold the dn_dbufs_mtx while we get the new dbuf
1721          * in the hash table *and* added to the dbufs list.
1722          * This prevents a possible deadlock with someone
1723          * trying to look up this dbuf before its added to the
1724          * dn_dbufs list.
1725          */
1726         mutex_enter(&dn->dn_dbufs_mtx);
1727         db->db_state = DB_EVICTING;
1728         if ((odb = dbuf_hash_insert(db)) != NULL) {
1729                 /* someone else inserted it first */
1730                 kmem_cache_free(dbuf_cache, db);
1731                 mutex_exit(&dn->dn_dbufs_mtx);
1732                 return (odb);
1733         }
1734         list_insert_head(&dn->dn_dbufs, db);
1735         db->db_state = DB_UNCACHED;
1736         mutex_exit(&dn->dn_dbufs_mtx);
1737         arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1738 
1739         if (parent && parent != dn->dn_dbuf)
1740                 dbuf_add_ref(parent, db);
1741 
1742         ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1743             refcount_count(&dn->dn_holds) > 0);
1744         (void) refcount_add(&dn->dn_holds, db);
1745         (void) atomic_inc_32_nv(&dn->dn_dbufs_count);
1746 
1747         dprintf_dbuf(db, "db=%p\n", db);
1748 
1749         return (db);
1750 }
1751 
1752 static int
1753 dbuf_do_evict(void *private)
1754 {
1755         arc_buf_t *buf = private;
1756         dmu_buf_impl_t *db = buf->b_private;
1757 
1758         if (!MUTEX_HELD(&db->db_mtx))
1759                 mutex_enter(&db->db_mtx);
1760 
1761         ASSERT(refcount_is_zero(&db->db_holds));
1762 
1763         if (db->db_state != DB_EVICTING) {
1764                 ASSERT(db->db_state == DB_CACHED);
1765                 DBUF_VERIFY(db);
1766                 db->db_buf = NULL;
1767                 dbuf_evict(db);
1768         } else {
1769                 mutex_exit(&db->db_mtx);
1770                 dbuf_destroy(db);
1771         }
1772         return (0);
1773 }
1774 
1775 static void
1776 dbuf_destroy(dmu_buf_impl_t *db)
1777 {
1778         ASSERT(refcount_is_zero(&db->db_holds));
1779 
1780         if (db->db_blkid != DMU_BONUS_BLKID) {
1781                 /*
1782                  * If this dbuf is still on the dn_dbufs list,
1783                  * remove it from that list.
1784                  */
1785                 if (db->db_dnode_handle != NULL) {
1786                         dnode_t *dn;
1787 
1788                         DB_DNODE_ENTER(db);
1789                         dn = DB_DNODE(db);
1790                         mutex_enter(&dn->dn_dbufs_mtx);
1791                         list_remove(&dn->dn_dbufs, db);
1792                         (void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1793                         mutex_exit(&dn->dn_dbufs_mtx);
1794                         DB_DNODE_EXIT(db);
1795                         /*
1796                          * Decrementing the dbuf count means that the hold
1797                          * corresponding to the removed dbuf is no longer
1798                          * discounted in dnode_move(), so the dnode cannot be
1799                          * moved until after we release the hold.
1800                          */
1801                         dnode_rele(dn, db);
1802                         db->db_dnode_handle = NULL;
1803                 }
1804                 dbuf_hash_remove(db);
1805         }
1806         db->db_parent = NULL;
1807         db->db_buf = NULL;
1808 
1809         ASSERT(!list_link_active(&db->db_link));
1810         ASSERT(db->db.db_data == NULL);
1811         ASSERT(db->db_hash_next == NULL);
1812         ASSERT(db->db_blkptr == NULL);
1813         ASSERT(db->db_data_pending == NULL);
1814 
1815         kmem_cache_free(dbuf_cache, db);
1816         arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1817 }
1818 
1819 void
1820 dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1821 {
1822         dmu_buf_impl_t *db = NULL;
1823         blkptr_t *bp = NULL;
1824 
1825         ASSERT(blkid != DMU_BONUS_BLKID);
1826         ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1827 
1828         if (dnode_block_freed(dn, blkid))
1829                 return;
1830 
1831         /* dbuf_find() returns with db_mtx held */
1832         if (db = dbuf_find(dn, 0, blkid)) {
1833                 /*
1834                  * This dbuf is already in the cache.  We assume that
1835                  * it is already CACHED, or else about to be either
1836                  * read or filled.
1837                  */
1838                 mutex_exit(&db->db_mtx);
1839                 return;
1840         }
1841 
1842         if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
1843                 if (bp && !BP_IS_HOLE(bp)) {
1844                         int priority = dn->dn_type == DMU_OT_DDT_ZAP ?
1845                             ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ;
1846                         arc_buf_t *pbuf;
1847                         dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
1848                         uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
1849                         zbookmark_t zb;
1850 
1851                         SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
1852                             dn->dn_object, 0, blkid);
1853 
1854                         if (db)
1855                                 pbuf = db->db_buf;
1856                         else
1857                                 pbuf = dn->dn_objset->os_phys_buf;
1858 
1859                         (void) dsl_read(NULL, dn->dn_objset->os_spa,
1860                             bp, pbuf, NULL, NULL, priority,
1861                             ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
1862                             &aflags, &zb);
1863                 }
1864                 if (db)
1865                         dbuf_rele(db, NULL);
1866         }
1867 }
1868 
1869 /*
1870  * Returns with db_holds incremented, and db_mtx not held.
1871  * Note: dn_struct_rwlock must be held.
1872  */
1873 int
1874 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1875     void *tag, dmu_buf_impl_t **dbp)
1876 {
1877         dmu_buf_impl_t *db, *parent = NULL;
1878 
1879         ASSERT(blkid != DMU_BONUS_BLKID);
1880         ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1881         ASSERT3U(dn->dn_nlevels, >, level);
1882 
1883         *dbp = NULL;
1884 top:
1885         /* dbuf_find() returns with db_mtx held */
1886         db = dbuf_find(dn, level, blkid);
1887 
1888         if (db == NULL) {
1889                 blkptr_t *bp = NULL;
1890                 int err;
1891 
1892                 ASSERT3P(parent, ==, NULL);
1893                 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1894                 if (fail_sparse) {
1895                         if (err == 0 && bp && BP_IS_HOLE(bp))
1896                                 err = ENOENT;
1897                         if (err) {
1898                                 if (parent)
1899                                         dbuf_rele(parent, NULL);
1900                                 return (err);
1901                         }
1902                 }
1903                 if (err && err != ENOENT)
1904                         return (err);
1905                 db = dbuf_create(dn, level, blkid, parent, bp);
1906         }
1907 
1908         if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1909                 arc_buf_add_ref(db->db_buf, db);
1910                 if (db->db_buf->b_data == NULL) {
1911                         dbuf_clear(db);
1912                         if (parent) {
1913                                 dbuf_rele(parent, NULL);
1914                                 parent = NULL;
1915                         }
1916                         goto top;
1917                 }
1918                 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1919         }
1920 
1921         ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1922 
1923         /*
1924          * If this buffer is currently syncing out, and we are are
1925          * still referencing it from db_data, we need to make a copy
1926          * of it in case we decide we want to dirty it again in this txg.
1927          */
1928         if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1929             dn->dn_object != DMU_META_DNODE_OBJECT &&
1930             db->db_state == DB_CACHED && db->db_data_pending) {
1931                 dbuf_dirty_record_t *dr = db->db_data_pending;
1932 
1933                 if (dr->dt.dl.dr_data == db->db_buf) {
1934                         arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1935 
1936                         dbuf_set_data(db,
1937                             arc_buf_alloc(dn->dn_objset->os_spa,
1938                             db->db.db_size, db, type));
1939                         bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
1940                             db->db.db_size);
1941                 }
1942         }
1943 
1944         (void) refcount_add(&db->db_holds, tag);
1945         dbuf_update_data(db);
1946         DBUF_VERIFY(db);
1947         mutex_exit(&db->db_mtx);
1948 
1949         /* NOTE: we can't rele the parent until after we drop the db_mtx */
1950         if (parent)
1951                 dbuf_rele(parent, NULL);
1952 
1953         ASSERT3P(DB_DNODE(db), ==, dn);
1954         ASSERT3U(db->db_blkid, ==, blkid);
1955         ASSERT3U(db->db_level, ==, level);
1956         *dbp = db;
1957 
1958         return (0);
1959 }
1960 
1961 dmu_buf_impl_t *
1962 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1963 {
1964         dmu_buf_impl_t *db;
1965         int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1966         return (err ? NULL : db);
1967 }
1968 
1969 dmu_buf_impl_t *
1970 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1971 {
1972         dmu_buf_impl_t *db;
1973         int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1974         return (err ? NULL : db);
1975 }
1976 
1977 void
1978 dbuf_create_bonus(dnode_t *dn)
1979 {
1980         ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1981 
1982         ASSERT(dn->dn_bonus == NULL);
1983         dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL);
1984 }
1985 
1986 int
1987 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
1988 {
1989         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1990         dnode_t *dn;
1991 
1992         if (db->db_blkid != DMU_SPILL_BLKID)
1993                 return (ENOTSUP);
1994         if (blksz == 0)
1995                 blksz = SPA_MINBLOCKSIZE;
1996         if (blksz > SPA_MAXBLOCKSIZE)
1997                 blksz = SPA_MAXBLOCKSIZE;
1998         else
1999                 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
2000 
2001         DB_DNODE_ENTER(db);
2002         dn = DB_DNODE(db);
2003         rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2004         dbuf_new_size(db, blksz, tx);
2005         rw_exit(&dn->dn_struct_rwlock);
2006         DB_DNODE_EXIT(db);
2007 
2008         return (0);
2009 }
2010 
2011 void
2012 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx)
2013 {
2014         dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
2015 }
2016 
2017 #pragma weak dmu_buf_add_ref = dbuf_add_ref
2018 void
2019 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2020 {
2021         int64_t holds = refcount_add(&db->db_holds, tag);
2022         ASSERT(holds > 1);
2023 }
2024 
2025 /*
2026  * If you call dbuf_rele() you had better not be referencing the dnode handle
2027  * unless you have some other direct or indirect hold on the dnode. (An indirect
2028  * hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
2029  * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
2030  * dnode's parent dbuf evicting its dnode handles.
2031  */
2032 #pragma weak dmu_buf_rele = dbuf_rele
2033 void
2034 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2035 {
2036         mutex_enter(&db->db_mtx);
2037         dbuf_rele_and_unlock(db, tag);
2038 }
2039 
2040 /*
2041  * dbuf_rele() for an already-locked dbuf.  This is necessary to allow
2042  * db_dirtycnt and db_holds to be updated atomically.
2043  */
2044 void
2045 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2046 {
2047         int64_t holds;
2048 
2049         ASSERT(MUTEX_HELD(&db->db_mtx));
2050         DBUF_VERIFY(db);
2051 
2052         /*
2053          * Remove the reference to the dbuf before removing its hold on the
2054          * dnode so we can guarantee in dnode_move() that a referenced bonus
2055          * buffer has a corresponding dnode hold.
2056          */
2057         holds = refcount_remove(&db->db_holds, tag);
2058         ASSERT(holds >= 0);
2059 
2060         /*
2061          * We can't freeze indirects if there is a possibility that they
2062          * may be modified in the current syncing context.
2063          */
2064         if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2065                 arc_buf_freeze(db->db_buf);
2066 
2067         if (holds == db->db_dirtycnt &&
2068             db->db_level == 0 && db->db_immediate_evict)
2069                 dbuf_evict_user(db);
2070 
2071         if (holds == 0) {
2072                 if (db->db_blkid == DMU_BONUS_BLKID) {
2073                         mutex_exit(&db->db_mtx);
2074 
2075                         /*
2076                          * If the dnode moves here, we cannot cross this barrier
2077                          * until the move completes.
2078                          */
2079                         DB_DNODE_ENTER(db);
2080                         (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count);
2081                         DB_DNODE_EXIT(db);
2082                         /*
2083                          * The bonus buffer's dnode hold is no longer discounted
2084                          * in dnode_move(). The dnode cannot move until after
2085                          * the dnode_rele().
2086                          */
2087                         dnode_rele(DB_DNODE(db), db);
2088                 } else if (db->db_buf == NULL) {
2089                         /*
2090                          * This is a special case: we never associated this
2091                          * dbuf with any data allocated from the ARC.
2092                          */
2093                         ASSERT(db->db_state == DB_UNCACHED ||
2094                             db->db_state == DB_NOFILL);
2095                         dbuf_evict(db);
2096                 } else if (arc_released(db->db_buf)) {
2097                         arc_buf_t *buf = db->db_buf;
2098                         /*
2099                          * This dbuf has anonymous data associated with it.
2100                          */
2101                         dbuf_set_data(db, NULL);
2102                         VERIFY(arc_buf_remove_ref(buf, db) == 1);
2103                         dbuf_evict(db);
2104                 } else {
2105                         VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
2106 
2107                         /*
2108                          * A dbuf will be eligible for eviction if either the
2109                          * 'primarycache' property is set or a duplicate
2110                          * copy of this buffer is already cached in the arc.
2111                          *
2112                          * In the case of the 'primarycache' a buffer
2113                          * is considered for eviction if it matches the
2114                          * criteria set in the property.
2115                          *
2116                          * To decide if our buffer is considered a
2117                          * duplicate, we must call into the arc to determine
2118                          * if multiple buffers are referencing the same
2119                          * block on-disk. If so, then we simply evict
2120                          * ourselves.
2121                          */
2122                         if (!DBUF_IS_CACHEABLE(db) ||
2123                             arc_buf_eviction_needed(db->db_buf))
2124                                 dbuf_clear(db);
2125                         else
2126                                 mutex_exit(&db->db_mtx);
2127                 }
2128         } else {
2129                 mutex_exit(&db->db_mtx);
2130         }
2131 }
2132 
2133 #pragma weak dmu_buf_refcount = dbuf_refcount
2134 uint64_t
2135 dbuf_refcount(dmu_buf_impl_t *db)
2136 {
2137         return (refcount_count(&db->db_holds));
2138 }
2139 
2140 void *
2141 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2142     dmu_buf_evict_func_t *evict_func)
2143 {
2144         return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2145             user_data_ptr_ptr, evict_func));
2146 }
2147 
2148 void *
2149 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2150     dmu_buf_evict_func_t *evict_func)
2151 {
2152         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2153 
2154         db->db_immediate_evict = TRUE;
2155         return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2156             user_data_ptr_ptr, evict_func));
2157 }
2158 
2159 void *
2160 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
2161     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
2162 {
2163         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2164         ASSERT(db->db_level == 0);
2165 
2166         ASSERT((user_ptr == NULL) == (evict_func == NULL));
2167 
2168         mutex_enter(&db->db_mtx);
2169 
2170         if (db->db_user_ptr == old_user_ptr) {
2171                 db->db_user_ptr = user_ptr;
2172                 db->db_user_data_ptr_ptr = user_data_ptr_ptr;
2173                 db->db_evict_func = evict_func;
2174 
2175                 dbuf_update_data(db);
2176         } else {
2177                 old_user_ptr = db->db_user_ptr;
2178         }
2179 
2180         mutex_exit(&db->db_mtx);
2181         return (old_user_ptr);
2182 }
2183 
2184 void *
2185 dmu_buf_get_user(dmu_buf_t *db_fake)
2186 {
2187         dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2188         ASSERT(!refcount_is_zero(&db->db_holds));
2189 
2190         return (db->db_user_ptr);
2191 }
2192 
2193 boolean_t
2194 dmu_buf_freeable(dmu_buf_t *dbuf)
2195 {
2196         boolean_t res = B_FALSE;
2197         dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2198 
2199         if (db->db_blkptr)
2200                 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2201                     db->db_blkptr, db->db_blkptr->blk_birth);
2202 
2203         return (res);
2204 }
2205 
2206 blkptr_t *
2207 dmu_buf_get_blkptr(dmu_buf_t *db)
2208 {
2209         dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2210         return (dbi->db_blkptr);
2211 }
2212 
2213 static void
2214 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2215 {
2216         /* ASSERT(dmu_tx_is_syncing(tx) */
2217         ASSERT(MUTEX_HELD(&db->db_mtx));
2218 
2219         if (db->db_blkptr != NULL)
2220                 return;
2221 
2222         if (db->db_blkid == DMU_SPILL_BLKID) {
2223                 db->db_blkptr = &dn->dn_phys->dn_spill;
2224                 BP_ZERO(db->db_blkptr);
2225                 return;
2226         }
2227         if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2228                 /*
2229                  * This buffer was allocated at a time when there was
2230                  * no available blkptrs from the dnode, or it was
2231                  * inappropriate to hook it in (i.e., nlevels mis-match).
2232                  */
2233                 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2234                 ASSERT(db->db_parent == NULL);
2235                 db->db_parent = dn->dn_dbuf;
2236                 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2237                 DBUF_VERIFY(db);
2238         } else {
2239                 dmu_buf_impl_t *parent = db->db_parent;
2240                 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2241 
2242                 ASSERT(dn->dn_phys->dn_nlevels > 1);
2243                 if (parent == NULL) {
2244                         mutex_exit(&db->db_mtx);
2245                         rw_enter(&dn->dn_struct_rwlock, RW_READER);
2246                         (void) dbuf_hold_impl(dn, db->db_level+1,
2247                             db->db_blkid >> epbs, FALSE, db, &parent);
2248                         rw_exit(&dn->dn_struct_rwlock);
2249                         mutex_enter(&db->db_mtx);
2250                         db->db_parent = parent;
2251                 }
2252                 db->db_blkptr = (blkptr_t *)parent->db.db_data +
2253                     (db->db_blkid & ((1ULL << epbs) - 1));
2254                 DBUF_VERIFY(db);
2255         }
2256 }
2257 
2258 static void
2259 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2260 {
2261         dmu_buf_impl_t *db = dr->dr_dbuf;
2262         dnode_t *dn;
2263         zio_t *zio;
2264 
2265         ASSERT(dmu_tx_is_syncing(tx));
2266 
2267         dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2268 
2269         mutex_enter(&db->db_mtx);
2270 
2271         ASSERT(db->db_level > 0);
2272         DBUF_VERIFY(db);
2273 
2274         if (db->db_buf == NULL) {
2275                 mutex_exit(&db->db_mtx);
2276                 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2277                 mutex_enter(&db->db_mtx);
2278         }
2279         ASSERT3U(db->db_state, ==, DB_CACHED);
2280         ASSERT(db->db_buf != NULL);
2281 
2282         DB_DNODE_ENTER(db);
2283         dn = DB_DNODE(db);
2284         ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2285         dbuf_check_blkptr(dn, db);
2286         DB_DNODE_EXIT(db);
2287 
2288         db->db_data_pending = dr;
2289 
2290         mutex_exit(&db->db_mtx);
2291         dbuf_write(dr, db->db_buf, tx);
2292 
2293         zio = dr->dr_zio;
2294         mutex_enter(&dr->dt.di.dr_mtx);
2295         dbuf_sync_list(&dr->dt.di.dr_children, tx);
2296         ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2297         mutex_exit(&dr->dt.di.dr_mtx);
2298         zio_nowait(zio);
2299 }
2300 
2301 static void
2302 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2303 {
2304         arc_buf_t **datap = &dr->dt.dl.dr_data;
2305         dmu_buf_impl_t *db = dr->dr_dbuf;
2306         dnode_t *dn;
2307         objset_t *os;
2308         uint64_t txg = tx->tx_txg;
2309 
2310         ASSERT(dmu_tx_is_syncing(tx));
2311 
2312         dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2313 
2314         mutex_enter(&db->db_mtx);
2315         /*
2316          * To be synced, we must be dirtied.  But we
2317          * might have been freed after the dirty.
2318          */
2319         if (db->db_state == DB_UNCACHED) {
2320                 /* This buffer has been freed since it was dirtied */
2321                 ASSERT(db->db.db_data == NULL);
2322         } else if (db->db_state == DB_FILL) {
2323                 /* This buffer was freed and is now being re-filled */
2324                 ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2325         } else {
2326                 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2327         }
2328         DBUF_VERIFY(db);
2329 
2330         DB_DNODE_ENTER(db);
2331         dn = DB_DNODE(db);
2332 
2333         if (db->db_blkid == DMU_SPILL_BLKID) {
2334                 mutex_enter(&dn->dn_mtx);
2335                 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR;
2336                 mutex_exit(&dn->dn_mtx);
2337         }
2338 
2339         /*
2340          * If this is a bonus buffer, simply copy the bonus data into the
2341          * dnode.  It will be written out when the dnode is synced (and it
2342          * will be synced, since it must have been dirty for dbuf_sync to
2343          * be called).
2344          */
2345         if (db->db_blkid == DMU_BONUS_BLKID) {
2346                 dbuf_dirty_record_t **drp;
2347 
2348                 ASSERT(*datap != NULL);
2349                 ASSERT0(db->db_level);
2350                 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
2351                 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
2352                 DB_DNODE_EXIT(db);
2353 
2354                 if (*datap != db->db.db_data) {
2355                         zio_buf_free(*datap, DN_MAX_BONUSLEN);
2356                         arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
2357                 }
2358                 db->db_data_pending = NULL;
2359                 drp = &db->db_last_dirty;
2360                 while (*drp != dr)
2361                         drp = &(*drp)->dr_next;
2362                 ASSERT(dr->dr_next == NULL);
2363                 ASSERT(dr->dr_dbuf == db);
2364                 *drp = dr->dr_next;
2365                 kmem_free(dr, sizeof (dbuf_dirty_record_t));
2366                 ASSERT(db->db_dirtycnt > 0);
2367                 db->db_dirtycnt -= 1;
2368                 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2369                 return;
2370         }
2371 
2372         os = dn->dn_objset;
2373 
2374         /*
2375          * This function may have dropped the db_mtx lock allowing a dmu_sync
2376          * operation to sneak in. As a result, we need to ensure that we
2377          * don't check the dr_override_state until we have returned from
2378          * dbuf_check_blkptr.
2379          */
2380         dbuf_check_blkptr(dn, db);
2381 
2382         /*
2383          * If this buffer is in the middle of an immediate write,
2384          * wait for the synchronous IO to complete.
2385          */
2386         while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
2387                 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
2388                 cv_wait(&db->db_changed, &db->db_mtx);
2389                 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
2390         }
2391 
2392         if (db->db_state != DB_NOFILL &&
2393             dn->dn_object != DMU_META_DNODE_OBJECT &&
2394             refcount_count(&db->db_holds) > 1 &&
2395             dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
2396             *datap == db->db_buf) {
2397                 /*
2398                  * If this buffer is currently "in use" (i.e., there
2399                  * are active holds and db_data still references it),
2400                  * then make a copy before we start the write so that
2401                  * any modifications from the open txg will not leak
2402                  * into this write.
2403                  *
2404                  * NOTE: this copy does not need to be made for
2405                  * objects only modified in the syncing context (e.g.
2406                  * DNONE_DNODE blocks).
2407                  */
2408                 int blksz = arc_buf_size(*datap);
2409                 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2410                 *datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2411                 bcopy(db->db.db_data, (*datap)->b_data, blksz);
2412         }
2413         db->db_data_pending = dr;
2414 
2415         mutex_exit(&db->db_mtx);
2416 
2417         dbuf_write(dr, *datap, tx);
2418 
2419         ASSERT(!list_link_active(&dr->dr_dirty_node));
2420         if (dn->dn_object == DMU_META_DNODE_OBJECT) {
2421                 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2422                 DB_DNODE_EXIT(db);
2423         } else {
2424                 /*
2425                  * Although zio_nowait() does not "wait for an IO", it does
2426                  * initiate the IO. If this is an empty write it seems plausible
2427                  * that the IO could actually be completed before the nowait
2428                  * returns. We need to DB_DNODE_EXIT() first in case
2429                  * zio_nowait() invalidates the dbuf.
2430                  */
2431                 DB_DNODE_EXIT(db);
2432                 zio_nowait(dr->dr_zio);
2433         }
2434 }
2435 
2436 void
2437 dbuf_sync_list(list_t *list, dmu_tx_t *tx)
2438 {
2439         dbuf_dirty_record_t *dr;
2440 
2441         while (dr = list_head(list)) {
2442                 if (dr->dr_zio != NULL) {
2443                         /*
2444                          * If we find an already initialized zio then we
2445                          * are processing the meta-dnode, and we have finished.
2446                          * The dbufs for all dnodes are put back on the list
2447                          * during processing, so that we can zio_wait()
2448                          * these IOs after initiating all child IOs.
2449                          */
2450                         ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2451                             DMU_META_DNODE_OBJECT);
2452                         break;
2453                 }
2454                 list_remove(list, dr);
2455                 if (dr->dr_dbuf->db_level > 0)
2456                         dbuf_sync_indirect(dr, tx);
2457                 else
2458                         dbuf_sync_leaf(dr, tx);
2459         }
2460 }
2461 
2462 /* ARGSUSED */
2463 static void
2464 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2465 {
2466         dmu_buf_impl_t *db = vdb;
2467         dnode_t *dn;
2468         blkptr_t *bp = zio->io_bp;
2469         blkptr_t *bp_orig = &zio->io_bp_orig;
2470         spa_t *spa = zio->io_spa;
2471         int64_t delta;
2472         uint64_t fill = 0;
2473         int i;
2474 
2475         ASSERT(db->db_blkptr == bp);
2476 
2477         DB_DNODE_ENTER(db);
2478         dn = DB_DNODE(db);
2479         delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
2480         dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
2481         zio->io_prev_space_delta = delta;
2482 
2483         if (BP_IS_HOLE(bp)) {
2484                 ASSERT(bp->blk_fill == 0);
2485                 DB_DNODE_EXIT(db);
2486                 return;
2487         }
2488 
2489         ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
2490             BP_GET_TYPE(bp) == dn->dn_type) ||
2491             (db->db_blkid == DMU_SPILL_BLKID &&
2492             BP_GET_TYPE(bp) == dn->dn_bonustype));
2493         ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2494 
2495         mutex_enter(&db->db_mtx);
2496 
2497 #ifdef ZFS_DEBUG
2498         if (db->db_blkid == DMU_SPILL_BLKID) {
2499                 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2500                 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2501                     db->db_blkptr == &dn->dn_phys->dn_spill);
2502         }
2503 #endif
2504 
2505         if (db->db_level == 0) {
2506                 mutex_enter(&dn->dn_mtx);
2507                 if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2508                     db->db_blkid != DMU_SPILL_BLKID)
2509                         dn->dn_phys->dn_maxblkid = db->db_blkid;
2510                 mutex_exit(&dn->dn_mtx);
2511 
2512                 if (dn->dn_type == DMU_OT_DNODE) {
2513                         dnode_phys_t *dnp = db->db.db_data;
2514                         for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2515                             i--, dnp++) {
2516                                 if (dnp->dn_type != DMU_OT_NONE)
2517                                         fill++;
2518                         }
2519                 } else {
2520                         fill = 1;
2521                 }
2522         } else {
2523                 blkptr_t *ibp = db->db.db_data;
2524                 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2525                 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2526                         if (BP_IS_HOLE(ibp))
2527                                 continue;
2528                         fill += ibp->blk_fill;
2529                 }
2530         }
2531         DB_DNODE_EXIT(db);
2532 
2533         bp->blk_fill = fill;
2534 
2535         mutex_exit(&db->db_mtx);
2536 }
2537 
2538 /* ARGSUSED */
2539 static void
2540 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2541 {
2542         dmu_buf_impl_t *db = vdb;
2543         blkptr_t *bp = zio->io_bp;
2544         blkptr_t *bp_orig = &zio->io_bp_orig;
2545         uint64_t txg = zio->io_txg;
2546         dbuf_dirty_record_t **drp, *dr;
2547 
2548         ASSERT0(zio->io_error);
2549         ASSERT(db->db_blkptr == bp);
2550 
2551         /*
2552          * For nopwrites and rewrites we ensure that the bp matches our
2553          * original and bypass all the accounting.
2554          */
2555         if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) {
2556                 ASSERT(BP_EQUAL(bp, bp_orig));
2557         } else {
2558                 objset_t *os;
2559                 dsl_dataset_t *ds;
2560                 dmu_tx_t *tx;
2561 
2562                 DB_GET_OBJSET(&os, db);
2563                 ds = os->os_dsl_dataset;
2564                 tx = os->os_synctx;
2565 
2566                 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
2567                 dsl_dataset_block_born(ds, bp, tx);
2568         }
2569 
2570         mutex_enter(&db->db_mtx);
2571 
2572         DBUF_VERIFY(db);
2573 
2574         drp = &db->db_last_dirty;
2575         while ((dr = *drp) != db->db_data_pending)
2576                 drp = &dr->dr_next;
2577         ASSERT(!list_link_active(&dr->dr_dirty_node));
2578         ASSERT(dr->dr_txg == txg);
2579         ASSERT(dr->dr_dbuf == db);
2580         ASSERT(dr->dr_next == NULL);
2581         *drp = dr->dr_next;
2582 
2583 #ifdef ZFS_DEBUG
2584         if (db->db_blkid == DMU_SPILL_BLKID) {
2585                 dnode_t *dn;
2586 
2587                 DB_DNODE_ENTER(db);
2588                 dn = DB_DNODE(db);
2589                 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2590                 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2591                     db->db_blkptr == &dn->dn_phys->dn_spill);
2592                 DB_DNODE_EXIT(db);
2593         }
2594 #endif
2595 
2596         if (db->db_level == 0) {
2597                 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2598                 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2599                 if (db->db_state != DB_NOFILL) {
2600                         if (dr->dt.dl.dr_data != db->db_buf)
2601                                 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
2602                                     db) == 1);
2603                         else if (!arc_released(db->db_buf))
2604                                 arc_set_callback(db->db_buf, dbuf_do_evict, db);
2605                 }
2606         } else {
2607                 dnode_t *dn;
2608 
2609                 DB_DNODE_ENTER(db);
2610                 dn = DB_DNODE(db);
2611                 ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2612                 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2613                 if (!BP_IS_HOLE(db->db_blkptr)) {
2614                         int epbs =
2615                             dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2616                         ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2617                             db->db.db_size);
2618                         ASSERT3U(dn->dn_phys->dn_maxblkid
2619                             >> (db->db_level * epbs), >=, db->db_blkid);
2620                         arc_set_callback(db->db_buf, dbuf_do_evict, db);
2621                 }
2622                 DB_DNODE_EXIT(db);
2623                 mutex_destroy(&dr->dt.di.dr_mtx);
2624                 list_destroy(&dr->dt.di.dr_children);
2625         }
2626         kmem_free(dr, sizeof (dbuf_dirty_record_t));
2627 
2628         cv_broadcast(&db->db_changed);
2629         ASSERT(db->db_dirtycnt > 0);
2630         db->db_dirtycnt -= 1;
2631         db->db_data_pending = NULL;
2632         dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2633 }
2634 
2635 static void
2636 dbuf_write_nofill_ready(zio_t *zio)
2637 {
2638         dbuf_write_ready(zio, NULL, zio->io_private);
2639 }
2640 
2641 static void
2642 dbuf_write_nofill_done(zio_t *zio)
2643 {
2644         dbuf_write_done(zio, NULL, zio->io_private);
2645 }
2646 
2647 static void
2648 dbuf_write_override_ready(zio_t *zio)
2649 {
2650         dbuf_dirty_record_t *dr = zio->io_private;
2651         dmu_buf_impl_t *db = dr->dr_dbuf;
2652 
2653         dbuf_write_ready(zio, NULL, db);
2654 }
2655 
2656 static void
2657 dbuf_write_override_done(zio_t *zio)
2658 {
2659         dbuf_dirty_record_t *dr = zio->io_private;
2660         dmu_buf_impl_t *db = dr->dr_dbuf;
2661         blkptr_t *obp = &dr->dt.dl.dr_overridden_by;
2662 
2663         mutex_enter(&db->db_mtx);
2664         if (!BP_EQUAL(zio->io_bp, obp)) {
2665                 if (!BP_IS_HOLE(obp))
2666                         dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
2667                 arc_release(dr->dt.dl.dr_data, db);
2668         }
2669         mutex_exit(&db->db_mtx);
2670 
2671         dbuf_write_done(zio, NULL, db);
2672 }
2673 
2674 static void
2675 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
2676 {
2677         dmu_buf_impl_t *db = dr->dr_dbuf;
2678         dnode_t *dn;
2679         objset_t *os;
2680         dmu_buf_impl_t *parent = db->db_parent;
2681         uint64_t txg = tx->tx_txg;
2682         zbookmark_t zb;
2683         zio_prop_t zp;
2684         zio_t *zio;
2685         int wp_flag = 0;
2686 
2687         DB_DNODE_ENTER(db);
2688         dn = DB_DNODE(db);
2689         os = dn->dn_objset;
2690 
2691         if (db->db_state != DB_NOFILL) {
2692                 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
2693                         /*
2694                          * Private object buffers are released here rather
2695                          * than in dbuf_dirty() since they are only modified
2696                          * in the syncing context and we don't want the
2697                          * overhead of making multiple copies of the data.
2698                          */
2699                         if (BP_IS_HOLE(db->db_blkptr)) {
2700                                 arc_buf_thaw(data);
2701                         } else {
2702                                 dbuf_release_bp(db);
2703                         }
2704                 }
2705         }
2706 
2707         if (parent != dn->dn_dbuf) {
2708                 ASSERT(parent && parent->db_data_pending);
2709                 ASSERT(db->db_level == parent->db_level-1);
2710                 ASSERT(arc_released(parent->db_buf));
2711                 zio = parent->db_data_pending->dr_zio;
2712         } else {
2713                 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
2714                     db->db_blkid != DMU_SPILL_BLKID) ||
2715                     (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
2716                 if (db->db_blkid != DMU_SPILL_BLKID)
2717                         ASSERT3P(db->db_blkptr, ==,
2718                             &dn->dn_phys->dn_blkptr[db->db_blkid]);
2719                 zio = dn->dn_zio;
2720         }
2721 
2722         ASSERT(db->db_level == 0 || data == db->db_buf);
2723         ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
2724         ASSERT(zio);
2725 
2726         SET_BOOKMARK(&zb, os->os_dsl_dataset ?
2727             os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
2728             db->db.db_object, db->db_level, db->db_blkid);
2729 
2730         if (db->db_blkid == DMU_SPILL_BLKID)
2731                 wp_flag = WP_SPILL;
2732         wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
2733 
2734         dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
2735         DB_DNODE_EXIT(db);
2736 
2737         if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
2738                 ASSERT(db->db_state != DB_NOFILL);
2739                 dr->dr_zio = zio_write(zio, os->os_spa, txg,
2740                     db->db_blkptr, data->b_data, arc_buf_size(data), &zp,
2741                     dbuf_write_override_ready, dbuf_write_override_done, dr,
2742                     ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2743                 mutex_enter(&db->db_mtx);
2744                 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
2745                 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
2746                     dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite);
2747                 mutex_exit(&db->db_mtx);
2748         } else if (db->db_state == DB_NOFILL) {
2749                 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF);
2750                 dr->dr_zio = zio_write(zio, os->os_spa, txg,
2751                     db->db_blkptr, NULL, db->db.db_size, &zp,
2752                     dbuf_write_nofill_ready, dbuf_write_nofill_done, db,
2753                     ZIO_PRIORITY_ASYNC_WRITE,
2754                     ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb);
2755         } else {
2756                 ASSERT(arc_released(data));
2757                 dr->dr_zio = arc_write(zio, os->os_spa, txg,
2758                     db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp,
2759                     dbuf_write_ready, dbuf_write_done, db,
2760                     ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2761         }
2762 }