3752 want more verifiable dbuf user eviction
Submitted by:   Justin Gibbs <justing@spectralogic.com>
Submitted by:   Will Andrews <willa@spectralogic.com>

@@ -1297,13 +1297,13 @@
         return (0);
 }
 
 /*ARGSUSED*/
 void
-sa_evict(dmu_buf_t *db, void *sap)
+sa_evict(dmu_buf_user_t *dbu)
 {
-        panic("evicting sa dbuf %p\n", (void *)db);
+        panic("evicting sa dbuf\n");
 }
 
 static void
 sa_idx_tab_rele(objset_t *os, void *arg)
 {

@@ -1338,13 +1338,14 @@
 }
 
 void
 sa_handle_destroy(sa_handle_t *hdl)
 {
+        dmu_buf_t *db = hdl->sa_bonus;
+
         mutex_enter(&hdl->sa_lock);
-        (void) dmu_buf_update_user((dmu_buf_t *)hdl->sa_bonus, hdl,
-            NULL, NULL, NULL);
+        (void) dmu_buf_remove_user(db, &hdl->db_evict);
 
         if (hdl->sa_bonus_tab) {
                 sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
                 hdl->sa_bonus_tab = NULL;
         }

@@ -1366,37 +1367,42 @@
 sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
 {
         int error = 0;
         dmu_object_info_t doi;
-        sa_handle_t *handle;
+        sa_handle_t *handle = NULL;
 
 #ifdef ZFS_DEBUG
         dmu_object_info_from_db(db, &doi);
         ASSERT(doi.doi_bonus_type == DMU_OT_SA ||
             doi.doi_bonus_type == DMU_OT_ZNODE);
 #endif
         /* find handle, if it exists */
         /* if one doesn't exist then create a new one, and initialize it */
 
-        handle = (hdl_type == SA_HDL_SHARED) ? dmu_buf_get_user(db) : NULL;
+        if (hdl_type == SA_HDL_SHARED)
+                handle = (sa_handle_t *)dmu_buf_get_user(db);
+
         if (handle == NULL) {
-                sa_handle_t *newhandle;
+                sa_handle_t *winner = NULL;
+
+                bzero(&handle->db_evict, sizeof(dmu_buf_user_t));
                 handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
                 handle->sa_userp = userp;
                 handle->sa_bonus = db;
                 handle->sa_os = os;
                 handle->sa_spill = NULL;
 
                 error = sa_build_index(handle, SA_BONUS);
-                newhandle = (hdl_type == SA_HDL_SHARED) ?
-                    dmu_buf_set_user_ie(db, handle,
-                    NULL, sa_evict) : NULL;
-
-                if (newhandle != NULL) {
+                if (hdl_type == SA_HDL_SHARED) {
+                        dmu_buf_init_user(&handle->db_evict, sa_evict);
+                        winner = (sa_handle_t *)
+                            dmu_buf_set_user_ie(db, &handle->db_evict);
+                }
+                if (winner != NULL) {
                         kmem_cache_free(sa_cache, handle);
-                        handle = newhandle;
+                        handle = winner;
                 }
         }
         *handlepp = handle;
 
         return (error);

@@ -1905,12 +1911,16 @@
 }
 
 void
 sa_update_user(sa_handle_t *newhdl, sa_handle_t *oldhdl)
 {
-        (void) dmu_buf_update_user((dmu_buf_t *)newhdl->sa_bonus,
-            oldhdl, newhdl, NULL, sa_evict);
+        dmu_buf_user_t *new_user = &newhdl->db_evict;
+        dmu_buf_user_t *old_user = &oldhdl->db_evict;
+
+        dmu_buf_init_user(new_user, sa_evict);
+        VERIFY(dmu_buf_replace_user(newhdl->sa_bonus, old_user,
+            new_user) == old_user);
         oldhdl->sa_bonus = NULL;
 }
 
 void
 sa_set_userp(sa_handle_t *hdl, void *ptr)