Print this page
3752 want more verifiable dbuf user eviction
Submitted by: Justin Gibbs <justing@spectralogic.com>
Submitted by: Will Andrews <willa@spectralogic.com>
*** 202,224 ****
}
static arc_evict_func_t dbuf_do_evict;
static void
! dbuf_evict_user(dmu_buf_impl_t *db)
{
ASSERT(MUTEX_HELD(&db->db_mtx));
! if (db->db_level != 0 || db->db_evict_func == NULL)
return;
! if (db->db_user_data_ptr_ptr)
! *db->db_user_data_ptr_ptr = db->db.db_data;
! db->db_evict_func(&db->db, db->db_user_ptr);
! db->db_user_ptr = NULL;
! db->db_user_data_ptr_ptr = NULL;
! db->db_evict_func = NULL;
}
boolean_t
dbuf_is_metadata(dmu_buf_impl_t *db)
{
--- 202,342 ----
}
static arc_evict_func_t dbuf_do_evict;
static void
! dbuf_verify_user(dmu_buf_impl_t *db, boolean_t evicting)
! {
! #ifdef ZFS_DEBUG
!
! if (db->db_level != 0)
! ASSERT(db->db_user == NULL);
!
! if (db->db_user == NULL)
! return;
!
! /* Clients must resolve a dbuf before attaching user data. */
! ASSERT(db->db.db_data != NULL && db->db_state == DB_CACHED);
! /*
! * We can't check the hold count here, because they are modified
! * independently of the dbuf mutex. But it would be nice to ensure
! * that the user has the appropriate number.
! */
! #endif
! }
!
! /*
! * Evict the dbuf's user, either immediately, or use a provided queue.
! *
! * Call dmu_buf_process_user_evicts or dmu_buf_destroy_user_evict_list
! * on the list when finished generating it.
! *
! * NOTE: If db->db_immediate_evict is FALSE, evict_list_p must be provided.
! * NOTE: See dmu_buf_user_t about how this process works.
! */
! static void
! dbuf_evict_user(dmu_buf_impl_t *db, list_t *evict_list_p)
{
ASSERT(MUTEX_HELD(&db->db_mtx));
+ ASSERT(evict_list_p != NULL);
+ dbuf_verify_user(db, /*evicting*/B_TRUE);
! if (db->db_user == NULL)
return;
! ASSERT(!list_link_active(&db->db_user->evict_queue_link));
! list_insert_head(evict_list_p, db->db_user);
! db->db_user = NULL;
! }
!
! /*
! * Replace the current user of the dbuf. Requires that the caller knows who
! * the old user is. Returns the old user, which may not necessarily be
! * the same old_user provided by the caller.
! */
! dmu_buf_user_t *
! dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
! dmu_buf_user_t *new_user)
! {
! dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
!
! mutex_enter(&db->db_mtx);
! dbuf_verify_user(db, /*evicting*/B_FALSE);
! if (db->db_user == old_user)
! db->db_user = new_user;
! else
! old_user = db->db_user;
! dbuf_verify_user(db, /*evicting*/B_FALSE);
! mutex_exit(&db->db_mtx);
!
! return (old_user);
! }
!
! /*
! * Set the user eviction data for the DMU beturns NULL on success,
! * or the existing user if another user currently owns the buffer.
! */
! dmu_buf_user_t *
! dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
! {
! return (dmu_buf_replace_user(db_fake, NULL, user));
! }
!
! dmu_buf_user_t *
! dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
! {
! dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
!
! db->db_immediate_evict = TRUE;
! return (dmu_buf_set_user(db_fake, user));
! }
!
! /*
! * Remove the user eviction data for the DMU buffer.
! */
! dmu_buf_user_t *
! dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
! {
! return (dmu_buf_replace_user(db_fake, user, NULL));
! }
!
! /*
! * Returns the db_user set with dmu_buf_update_user(), or NULL if not set.
! */
! dmu_buf_user_t *
! dmu_buf_get_user(dmu_buf_t *db_fake)
! {
! dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
!
! dbuf_verify_user(db, /*evicting*/B_FALSE);
! return (db->db_user);
! }
!
! static void
! dbuf_clear_data(dmu_buf_impl_t *db, list_t *evict_list_p)
! {
! ASSERT(MUTEX_HELD(&db->db_mtx));
! ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
! dbuf_evict_user(db, evict_list_p);
! db->db_buf = NULL;
! db->db.db_data = NULL;
! if (db->db_state != DB_NOFILL)
! db->db_state = DB_UNCACHED;
! }
!
! static void
! dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
! {
! ASSERT(MUTEX_HELD(&db->db_mtx));
! ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
! ASSERT(buf != NULL);
!
! db->db_buf = buf;
! ASSERT(buf->b_data != NULL);
! db->db.db_data = buf->b_data;
! if (!arc_released(buf))
! arc_set_callback(buf, dbuf_do_evict, db);
}
boolean_t
dbuf_is_metadata(dmu_buf_impl_t *db)
{
*** 234,250 ****
return (is_metadata);
}
}
void
! dbuf_evict(dmu_buf_impl_t *db)
{
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(db->db_buf == NULL);
ASSERT(db->db_data_pending == NULL);
! dbuf_clear(db);
dbuf_destroy(db);
}
void
dbuf_init(void)
--- 352,368 ----
return (is_metadata);
}
}
void
! dbuf_evict(dmu_buf_impl_t *db, list_t *evict_list_p)
{
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(db->db_buf == NULL);
ASSERT(db->db_data_pending == NULL);
! dbuf_clear(db, evict_list_p);
dbuf_destroy(db);
}
void
dbuf_init(void)
*** 403,449 ****
}
DB_DNODE_EXIT(db);
}
#endif
- static void
- dbuf_update_data(dmu_buf_impl_t *db)
- {
- ASSERT(MUTEX_HELD(&db->db_mtx));
- if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
- ASSERT(!refcount_is_zero(&db->db_holds));
- *db->db_user_data_ptr_ptr = db->db.db_data;
- }
- }
-
- static void
- dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
- {
- ASSERT(MUTEX_HELD(&db->db_mtx));
- ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
- db->db_buf = buf;
- if (buf != NULL) {
- ASSERT(buf->b_data != NULL);
- db->db.db_data = buf->b_data;
- if (!arc_released(buf))
- arc_set_callback(buf, dbuf_do_evict, db);
- dbuf_update_data(db);
- } else {
- dbuf_evict_user(db);
- db->db.db_data = NULL;
- if (db->db_state != DB_NOFILL)
- db->db_state = DB_UNCACHED;
- }
- }
-
/*
* Loan out an arc_buf for read. Return the loaned arc_buf.
*/
arc_buf_t *
dbuf_loan_arcbuf(dmu_buf_impl_t *db)
{
arc_buf_t *abuf;
mutex_enter(&db->db_mtx);
if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
int blksz = db->db.db_size;
spa_t *spa;
--- 521,540 ----
}
DB_DNODE_EXIT(db);
}
#endif
/*
* Loan out an arc_buf for read. Return the loaned arc_buf.
*/
arc_buf_t *
dbuf_loan_arcbuf(dmu_buf_impl_t *db)
{
arc_buf_t *abuf;
+ list_t evict_list;
+
+ dmu_buf_create_user_evict_list(&evict_list);
mutex_enter(&db->db_mtx);
if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
int blksz = db->db.db_size;
spa_t *spa;
*** 453,465 ****
abuf = arc_loan_buf(spa, blksz);
bcopy(db->db.db_data, abuf->b_data, blksz);
} else {
abuf = db->db_buf;
arc_loan_inuse_buf(abuf, db);
! dbuf_set_data(db, NULL);
mutex_exit(&db->db_mtx);
}
return (abuf);
}
uint64_t
dbuf_whichblock(dnode_t *dn, uint64_t offset)
--- 544,557 ----
abuf = arc_loan_buf(spa, blksz);
bcopy(db->db.db_data, abuf->b_data, blksz);
} else {
abuf = db->db_buf;
arc_loan_inuse_buf(abuf, db);
! dbuf_clear_data(db, &evict_list);
mutex_exit(&db->db_mtx);
}
+ dmu_buf_destroy_user_evict_list(&evict_list);
return (abuf);
}
uint64_t
dbuf_whichblock(dnode_t *dn, uint64_t offset)
*** 532,542 ****
if (bonuslen < DN_MAX_BONUSLEN)
bzero(db->db.db_data, DN_MAX_BONUSLEN);
if (bonuslen)
bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
DB_DNODE_EXIT(db);
- dbuf_update_data(db);
db->db_state = DB_CACHED;
mutex_exit(&db->db_mtx);
return;
}
--- 624,633 ----
*** 665,676 ****
--- 756,771 ----
}
static void
dbuf_noread(dmu_buf_impl_t *db)
{
+ list_t evict_list;
+
ASSERT(!refcount_is_zero(&db->db_holds));
ASSERT(db->db_blkid != DMU_BONUS_BLKID);
+ dmu_buf_create_user_evict_list(&evict_list);
+
mutex_enter(&db->db_mtx);
while (db->db_state == DB_READ || db->db_state == DB_FILL)
cv_wait(&db->db_changed, &db->db_mtx);
if (db->db_state == DB_UNCACHED) {
arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
*** 680,694 ****
ASSERT(db->db.db_data == NULL);
DB_GET_SPA(&spa, db);
dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
db->db_state = DB_FILL;
} else if (db->db_state == DB_NOFILL) {
! dbuf_set_data(db, NULL);
} else {
ASSERT3U(db->db_state, ==, DB_CACHED);
}
mutex_exit(&db->db_mtx);
}
/*
* This is our just-in-time copy function. It makes a copy of
* buffers, that have been modified in a previous transaction
--- 775,790 ----
ASSERT(db->db.db_data == NULL);
DB_GET_SPA(&spa, db);
dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
db->db_state = DB_FILL;
} else if (db->db_state == DB_NOFILL) {
! dbuf_clear_data(db, &evict_list);
} else {
ASSERT3U(db->db_state, ==, DB_CACHED);
}
mutex_exit(&db->db_mtx);
+ dmu_buf_destroy_user_evict_list(&evict_list);
}
/*
* This is our just-in-time copy function. It makes a copy of
* buffers, that have been modified in a previous transaction
*** 701,711 ****
* Note that when we are called from dbuf_free_range() we do
* not put a hold on the buffer, we just traverse the active
* dbuf list for the dnode.
*/
static void
! dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
{
dbuf_dirty_record_t *dr = db->db_last_dirty;
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(db->db.db_data != NULL);
--- 797,807 ----
* Note that when we are called from dbuf_free_range() we do
* not put a hold on the buffer, we just traverse the active
* dbuf list for the dnode.
*/
static void
! dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg, list_t *evict_list_p)
{
dbuf_dirty_record_t *dr = db->db_last_dirty;
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(db->db.db_data != NULL);
*** 737,747 ****
DB_GET_SPA(&spa, db);
dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
} else {
! dbuf_set_data(db, NULL);
}
}
void
dbuf_unoverride(dbuf_dirty_record_t *dr)
--- 833,843 ----
DB_GET_SPA(&spa, db);
dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
} else {
! dbuf_clear_data(db, evict_list_p);
}
}
void
dbuf_unoverride(dbuf_dirty_record_t *dr)
*** 794,803 ****
--- 890,902 ----
dmu_buf_impl_t *db, *db_next;
uint64_t txg = tx->tx_txg;
int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
uint64_t first_l1 = start >> epbs;
uint64_t last_l1 = end >> epbs;
+ list_t evict_list;
+
+ dmu_buf_create_user_evict_list(&evict_list);
if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) {
end = dn->dn_maxblkid;
last_l1 = end >> epbs;
}
*** 847,857 ****
mutex_exit(&db->db_mtx);
continue;
}
if (refcount_count(&db->db_holds) == 0) {
ASSERT(db->db_buf);
! dbuf_clear(db);
continue;
}
/* The dbuf is referenced */
if (db->db_last_dirty != NULL) {
--- 946,956 ----
mutex_exit(&db->db_mtx);
continue;
}
if (refcount_count(&db->db_holds) == 0) {
ASSERT(db->db_buf);
! dbuf_clear(db, &evict_list);
continue;
}
/* The dbuf is referenced */
if (db->db_last_dirty != NULL) {
*** 872,882 ****
* This dbuf is not dirty in the open context.
* Either uncache it (if its not referenced in
* the open context) or reset its contents to
* empty.
*/
! dbuf_fix_old_data(db, txg);
}
}
/* clear the contents if its cached */
if (db->db_state == DB_CACHED) {
ASSERT(db->db.db_data != NULL);
--- 971,981 ----
* This dbuf is not dirty in the open context.
* Either uncache it (if its not referenced in
* the open context) or reset its contents to
* empty.
*/
! dbuf_fix_old_data(db, txg, &evict_list);
}
}
/* clear the contents if its cached */
if (db->db_state == DB_CACHED) {
ASSERT(db->db.db_data != NULL);
*** 884,895 ****
--- 983,996 ----
bzero(db->db.db_data, db->db.db_size);
arc_buf_freeze(db->db_buf);
}
mutex_exit(&db->db_mtx);
+ dmu_buf_process_user_evicts(&evict_list);
}
mutex_exit(&dn->dn_dbufs_mtx);
+ dmu_buf_destroy_user_evict_list(&evict_list);
}
static int
dbuf_block_freeable(dmu_buf_impl_t *db)
{
*** 994,1003 ****
--- 1095,1107 ----
objset_t *os;
dbuf_dirty_record_t **drp, *dr;
int drop_struct_lock = FALSE;
boolean_t do_free_accounting = B_FALSE;
int txgoff = tx->tx_txg & TXG_MASK;
+ list_t evict_list;
+
+ dmu_buf_create_user_evict_list(&evict_list);
ASSERT(tx->tx_txg != 0);
ASSERT(!refcount_is_zero(&db->db_holds));
DMU_TX_DIRTY_BUF(tx, db);
*** 1068,1077 ****
--- 1172,1182 ----
if (db->db.db_object != DMU_META_DNODE_OBJECT &&
db->db_state != DB_NOFILL)
arc_buf_thaw(db->db_buf);
}
mutex_exit(&db->db_mtx);
+ dmu_buf_destroy_user_evict_list(&evict_list);
return (dr);
}
/*
* Only valid if not already dirty.
*** 1122,1132 ****
if (db->db_level == 0) {
void *data_old = db->db_buf;
if (db->db_state != DB_NOFILL) {
if (db->db_blkid == DMU_BONUS_BLKID) {
! dbuf_fix_old_data(db, tx->tx_txg);
data_old = db->db.db_data;
} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
/*
* Release the data buffer from the cache so
* that we can modify it without impacting
--- 1227,1237 ----
if (db->db_level == 0) {
void *data_old = db->db_buf;
if (db->db_state != DB_NOFILL) {
if (db->db_blkid == DMU_BONUS_BLKID) {
! dbuf_fix_old_data(db, tx->tx_txg, &evict_list);
data_old = db->db.db_data;
} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
/*
* Release the data buffer from the cache so
* that we can modify it without impacting
*** 1135,1145 ****
* private objects are not released until the
* syncing state (since they are only modified
* then).
*/
arc_release(db->db_buf, db);
! dbuf_fix_old_data(db, tx->tx_txg);
data_old = db->db_buf;
}
ASSERT(data_old != NULL);
}
dr->dt.dl.dr_data = data_old;
--- 1240,1250 ----
* private objects are not released until the
* syncing state (since they are only modified
* then).
*/
arc_release(db->db_buf, db);
! dbuf_fix_old_data(db, tx->tx_txg, &evict_list);
data_old = db->db_buf;
}
ASSERT(data_old != NULL);
}
dr->dt.dl.dr_data = data_old;
*** 1173,1182 ****
--- 1278,1288 ----
dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
db->db_dirtycnt += 1;
ASSERT3U(db->db_dirtycnt, <=, 3);
mutex_exit(&db->db_mtx);
+ dmu_buf_destroy_user_evict_list(&evict_list);
if (db->db_blkid == DMU_BONUS_BLKID ||
db->db_blkid == DMU_SPILL_BLKID) {
mutex_enter(&dn->dn_mtx);
ASSERT(!list_link_active(&dr->dr_dirty_node));
*** 1267,1276 ****
--- 1373,1383 ----
dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
{
dnode_t *dn;
uint64_t txg = tx->tx_txg;
dbuf_dirty_record_t *dr, **drp;
+ list_t evict_list;
ASSERT(txg != 0);
ASSERT(db->db_blkid != DMU_BONUS_BLKID);
ASSERT0(db->db_level);
ASSERT(MUTEX_HELD(&db->db_mtx));
*** 1284,1293 ****
--- 1391,1402 ----
if (dr == NULL || dr->dr_txg < txg)
return (B_FALSE);
ASSERT(dr->dr_txg == txg);
ASSERT(dr->dr_dbuf == db);
+ dmu_buf_create_user_evict_list(&evict_list);
+
DB_DNODE_ENTER(db);
dn = DB_DNODE(db);
/*
* Note: This code will probably work even if there are concurrent
*** 1339,1354 ****
if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
arc_buf_t *buf = db->db_buf;
ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
! dbuf_set_data(db, NULL);
VERIFY(arc_buf_remove_ref(buf, db));
! dbuf_evict(db);
return (B_TRUE);
}
return (B_FALSE);
}
#pragma weak dmu_buf_will_dirty = dbuf_will_dirty
void
--- 1448,1465 ----
if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
arc_buf_t *buf = db->db_buf;
ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
! dbuf_clear_data(db, &evict_list);
VERIFY(arc_buf_remove_ref(buf, db));
! dbuf_evict(db, &evict_list);
! dmu_buf_destroy_user_evict_list(&evict_list);
return (B_TRUE);
}
+ dmu_buf_destroy_user_evict_list(&evict_list);
return (B_FALSE);
}
#pragma weak dmu_buf_will_dirty = dbuf_will_dirty
void
*** 1491,1511 ****
* Sometimes, though, we will get a mix of these two:
* DMU: dbuf_clear()->arc_buf_evict()
* ARC: dbuf_do_evict()->dbuf_destroy()
*/
void
! dbuf_clear(dmu_buf_impl_t *db)
{
dnode_t *dn;
dmu_buf_impl_t *parent = db->db_parent;
dmu_buf_impl_t *dndb;
int dbuf_gone = FALSE;
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(refcount_is_zero(&db->db_holds));
! dbuf_evict_user(db);
if (db->db_state == DB_CACHED) {
ASSERT(db->db.db_data != NULL);
if (db->db_blkid == DMU_BONUS_BLKID) {
zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
--- 1602,1622 ----
* Sometimes, though, we will get a mix of these two:
* DMU: dbuf_clear()->arc_buf_evict()
* ARC: dbuf_do_evict()->dbuf_destroy()
*/
void
! dbuf_clear(dmu_buf_impl_t *db, list_t *evict_list_p)
{
dnode_t *dn;
dmu_buf_impl_t *parent = db->db_parent;
dmu_buf_impl_t *dndb;
int dbuf_gone = FALSE;
ASSERT(MUTEX_HELD(&db->db_mtx));
ASSERT(refcount_is_zero(&db->db_holds));
! dbuf_evict_user(db, evict_list_p);
if (db->db_state == DB_CACHED) {
ASSERT(db->db.db_data != NULL);
if (db->db_blkid == DMU_BONUS_BLKID) {
zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
*** 1644,1656 ****
db->db_dirtycnt = 0;
db->db_dnode_handle = dn->dn_handle;
db->db_parent = parent;
db->db_blkptr = blkptr;
! db->db_user_ptr = NULL;
! db->db_user_data_ptr_ptr = NULL;
! db->db_evict_func = NULL;
db->db_immediate_evict = 0;
db->db_freed_in_flight = 0;
if (blkid == DMU_BONUS_BLKID) {
ASSERT3P(parent, ==, dn->dn_dbuf);
--- 1755,1765 ----
db->db_dirtycnt = 0;
db->db_dnode_handle = dn->dn_handle;
db->db_parent = parent;
db->db_blkptr = blkptr;
! db->db_user = NULL;
db->db_immediate_evict = 0;
db->db_freed_in_flight = 0;
if (blkid == DMU_BONUS_BLKID) {
ASSERT3P(parent, ==, dn->dn_dbuf);
*** 1709,1733 ****
static int
dbuf_do_evict(void *private)
{
arc_buf_t *buf = private;
dmu_buf_impl_t *db = buf->b_private;
if (!MUTEX_HELD(&db->db_mtx))
mutex_enter(&db->db_mtx);
ASSERT(refcount_is_zero(&db->db_holds));
if (db->db_state != DB_EVICTING) {
ASSERT(db->db_state == DB_CACHED);
DBUF_VERIFY(db);
db->db_buf = NULL;
! dbuf_evict(db);
} else {
mutex_exit(&db->db_mtx);
dbuf_destroy(db);
}
return (0);
}
static void
dbuf_destroy(dmu_buf_impl_t *db)
--- 1818,1846 ----
static int
dbuf_do_evict(void *private)
{
arc_buf_t *buf = private;
dmu_buf_impl_t *db = buf->b_private;
+ list_t evict_list;
+
+ dmu_buf_create_user_evict_list(&evict_list);
if (!MUTEX_HELD(&db->db_mtx))
mutex_enter(&db->db_mtx);
ASSERT(refcount_is_zero(&db->db_holds));
if (db->db_state != DB_EVICTING) {
ASSERT(db->db_state == DB_CACHED);
DBUF_VERIFY(db);
db->db_buf = NULL;
! dbuf_evict(db, &evict_list);
} else {
mutex_exit(&db->db_mtx);
dbuf_destroy(db);
}
+ dmu_buf_destroy_user_evict_list(&evict_list);
return (0);
}
static void
dbuf_destroy(dmu_buf_impl_t *db)
*** 1824,1838 ****
--- 1937,1954 ----
int
dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
void *tag, dmu_buf_impl_t **dbp)
{
dmu_buf_impl_t *db, *parent = NULL;
+ list_t evict_list;
ASSERT(blkid != DMU_BONUS_BLKID);
ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
ASSERT3U(dn->dn_nlevels, >, level);
+ dmu_buf_create_user_evict_list(&evict_list);
+
*dbp = NULL;
top:
/* dbuf_find() returns with db_mtx held */
db = dbuf_find(dn, level, blkid);
*** 1857,1867 ****
}
if (db->db_buf && refcount_is_zero(&db->db_holds)) {
arc_buf_add_ref(db->db_buf, db);
if (db->db_buf->b_data == NULL) {
! dbuf_clear(db);
if (parent) {
dbuf_rele(parent, NULL);
parent = NULL;
}
goto top;
--- 1973,1983 ----
}
if (db->db_buf && refcount_is_zero(&db->db_holds)) {
arc_buf_add_ref(db->db_buf, db);
if (db->db_buf->b_data == NULL) {
! dbuf_clear(db, &evict_list);
if (parent) {
dbuf_rele(parent, NULL);
parent = NULL;
}
goto top;
*** 1891,1904 ****
db->db.db_size);
}
}
(void) refcount_add(&db->db_holds, tag);
- dbuf_update_data(db);
DBUF_VERIFY(db);
mutex_exit(&db->db_mtx);
/* NOTE: we can't rele the parent until after we drop the db_mtx */
if (parent)
dbuf_rele(parent, NULL);
ASSERT3P(DB_DNODE(db), ==, dn);
--- 2007,2021 ----
db->db.db_size);
}
}
(void) refcount_add(&db->db_holds, tag);
DBUF_VERIFY(db);
mutex_exit(&db->db_mtx);
+ dmu_buf_destroy_user_evict_list(&evict_list);
+
/* NOTE: we can't rele the parent until after we drop the db_mtx */
if (parent)
dbuf_rele(parent, NULL);
ASSERT3P(DB_DNODE(db), ==, dn);
*** 1994,2007 ****
--- 2111,2127 ----
*/
void
dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
{
int64_t holds;
+ list_t evict_list;
ASSERT(MUTEX_HELD(&db->db_mtx));
DBUF_VERIFY(db);
+ dmu_buf_create_user_evict_list(&evict_list);
+
/*
* Remove the reference to the dbuf before removing its hold on the
* dnode so we can guarantee in dnode_move() that a referenced bonus
* buffer has a corresponding dnode hold.
*/
*** 2015,2025 ****
if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
arc_buf_freeze(db->db_buf);
if (holds == db->db_dirtycnt &&
db->db_level == 0 && db->db_immediate_evict)
! dbuf_evict_user(db);
if (holds == 0) {
if (db->db_blkid == DMU_BONUS_BLKID) {
mutex_exit(&db->db_mtx);
--- 2135,2145 ----
if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
arc_buf_freeze(db->db_buf);
if (holds == db->db_dirtycnt &&
db->db_level == 0 && db->db_immediate_evict)
! dbuf_evict_user(db, &evict_list);
if (holds == 0) {
if (db->db_blkid == DMU_BONUS_BLKID) {
mutex_exit(&db->db_mtx);
*** 2041,2059 ****
* This is a special case: we never associated this
* dbuf with any data allocated from the ARC.
*/
ASSERT(db->db_state == DB_UNCACHED ||
db->db_state == DB_NOFILL);
! dbuf_evict(db);
} else if (arc_released(db->db_buf)) {
arc_buf_t *buf = db->db_buf;
/*
* This dbuf has anonymous data associated with it.
*/
! dbuf_set_data(db, NULL);
VERIFY(arc_buf_remove_ref(buf, db));
! dbuf_evict(db);
} else {
VERIFY(!arc_buf_remove_ref(db->db_buf, db));
/*
* A dbuf will be eligible for eviction if either the
--- 2161,2179 ----
* This is a special case: we never associated this
* dbuf with any data allocated from the ARC.
*/
ASSERT(db->db_state == DB_UNCACHED ||
db->db_state == DB_NOFILL);
! dbuf_evict(db, &evict_list);
} else if (arc_released(db->db_buf)) {
arc_buf_t *buf = db->db_buf;
/*
* This dbuf has anonymous data associated with it.
*/
! dbuf_clear_data(db, &evict_list);
VERIFY(arc_buf_remove_ref(buf, db));
! dbuf_evict(db, &evict_list);
} else {
VERIFY(!arc_buf_remove_ref(db->db_buf, db));
/*
* A dbuf will be eligible for eviction if either the
*** 2070,2148 ****
* block on-disk. If so, then we simply evict
* ourselves.
*/
if (!DBUF_IS_CACHEABLE(db) ||
arc_buf_eviction_needed(db->db_buf))
! dbuf_clear(db);
else
mutex_exit(&db->db_mtx);
}
} else {
mutex_exit(&db->db_mtx);
}
}
#pragma weak dmu_buf_refcount = dbuf_refcount
uint64_t
dbuf_refcount(dmu_buf_impl_t *db)
{
return (refcount_count(&db->db_holds));
}
- void *
- dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
- dmu_buf_evict_func_t *evict_func)
- {
- return (dmu_buf_update_user(db_fake, NULL, user_ptr,
- user_data_ptr_ptr, evict_func));
- }
-
- void *
- dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
- dmu_buf_evict_func_t *evict_func)
- {
- dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
-
- db->db_immediate_evict = TRUE;
- return (dmu_buf_update_user(db_fake, NULL, user_ptr,
- user_data_ptr_ptr, evict_func));
- }
-
- void *
- dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
- void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
- {
- dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
- ASSERT(db->db_level == 0);
-
- ASSERT((user_ptr == NULL) == (evict_func == NULL));
-
- mutex_enter(&db->db_mtx);
-
- if (db->db_user_ptr == old_user_ptr) {
- db->db_user_ptr = user_ptr;
- db->db_user_data_ptr_ptr = user_data_ptr_ptr;
- db->db_evict_func = evict_func;
-
- dbuf_update_data(db);
- } else {
- old_user_ptr = db->db_user_ptr;
- }
-
- mutex_exit(&db->db_mtx);
- return (old_user_ptr);
- }
-
- void *
- dmu_buf_get_user(dmu_buf_t *db_fake)
- {
- dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
- ASSERT(!refcount_is_zero(&db->db_holds));
-
- return (db->db_user_ptr);
- }
-
boolean_t
dmu_buf_freeable(dmu_buf_t *dbuf)
{
boolean_t res = B_FALSE;
dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
--- 2190,2216 ----
* block on-disk. If so, then we simply evict
* ourselves.
*/
if (!DBUF_IS_CACHEABLE(db) ||
arc_buf_eviction_needed(db->db_buf))
! dbuf_clear(db, &evict_list);
else
mutex_exit(&db->db_mtx);
}
} else {
mutex_exit(&db->db_mtx);
}
+ dmu_buf_destroy_user_evict_list(&evict_list);
}
#pragma weak dmu_buf_refcount = dbuf_refcount
uint64_t
dbuf_refcount(dmu_buf_impl_t *db)
{
return (refcount_count(&db->db_holds));
}
boolean_t
dmu_buf_freeable(dmu_buf_t *dbuf)
{
boolean_t res = B_FALSE;
dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;