Print this page
3740 Poor ZFS send / receive performance due to snapshot hold / release processing
Submitted by: Steven Hartland <steven.hartland@multiplay.co.uk>

@@ -34,13 +34,16 @@
 #include <sys/dsl_dir.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/zap.h>
 
 typedef struct dsl_dataset_user_hold_arg {
+        spa_t *dduha_spa;
         nvlist_t *dduha_holds;
+        nvlist_t *dduha_tmpholds;
         nvlist_t *dduha_errlist;
         minor_t dduha_minor;
+        boolean_t dduha_holds_created;
 } dsl_dataset_user_hold_arg_t;
 
 /*
  * If you add new checks here, you may need to add additional checks to the
  * "temporary" case in snapshot_check() in dmu_objset.c.

@@ -82,14 +85,17 @@
 {
         dsl_dataset_user_hold_arg_t *dduha = arg;
         dsl_pool_t *dp = dmu_tx_pool(tx);
         nvpair_t *pair;
         int rv = 0;
+        boolean_t holds_possible;
 
         if (spa_version(dp->dp_spa) < SPA_VERSION_USERREFS)
                 return (SET_ERROR(ENOTSUP));
 
+        holds_possible = B_FALSE;
+
         for (pair = nvlist_next_nvpair(dduha->dduha_holds, NULL); pair != NULL;
             pair = nvlist_next_nvpair(dduha->dduha_holds, pair)) {
                 int error = 0;
                 dsl_dataset_t *ds;
                 char *htag;

@@ -101,29 +107,56 @@
                 if (error == 0)
                         error = nvpair_value_string(pair, &htag);
                 if (error == 0) {
                         error = dsl_dataset_hold(dp,
                             nvpair_name(pair), FTAG, &ds);
+
+                        if (error == ENOENT) {
+                                /*
+                                 * We register ENOENT errors so they can be
+                                 * correctly reported if needed, such as when
+                                 * all holds fail.
+                                 */
+                                if (dduha->dduha_errlist != NULL) {
+                                        fnvlist_add_int32(dduha->dduha_errlist,
+                                            nvpair_name(pair), error);
+                                }
+                                continue;
+                        }
                 }
                 if (error == 0) {
                         error = dsl_dataset_user_hold_check_one(ds, htag,
                             dduha->dduha_minor != 0, tx);
                         dsl_dataset_rele(ds, FTAG);
                 }
 
                 if (error != 0) {
-                        rv = error;
+                        if (dduha->dduha_errlist != NULL) {
                         fnvlist_add_int32(dduha->dduha_errlist,
                             nvpair_name(pair), error);
                 }
+                        rv = error;
+                } else {
+                        holds_possible = B_TRUE;
+                }
         }
+
+        /*
+         * Check that at least one hold will possibly be created,
+         * otherwise fail.
+         */
+        if (rv == 0 && !holds_possible)
+                rv = ENOENT;
+
         return (rv);
 }
 
-void
-dsl_dataset_user_hold_sync_one(dsl_dataset_t *ds, const char *htag,
-    minor_t minor, uint64_t now, dmu_tx_t *tx)
+
+static void
+dsl_dataset_user_hold_sync_one_impl(dsl_dataset_user_hold_arg_t *dduha,
+    dsl_dataset_t *ds, const char *htag, minor_t minor, uint64_t now,
+    dmu_tx_t *tx)
 {
         dsl_pool_t *dp = ds->ds_dir->dd_pool;
         objset_t *mos = dp->dp_meta_objset;
         uint64_t zapobj;
 

@@ -143,20 +176,106 @@
         mutex_exit(&ds->ds_lock);
 
         VERIFY0(zap_add(mos, zapobj, htag, 8, 1, &now, tx));
 
         if (minor != 0) {
+                char name[MAXNAMELEN];
+                nvlist_t *tags;
+
                 VERIFY0(dsl_pool_user_hold(dp, ds->ds_object,
                     htag, now, tx));
-                dsl_register_onexit_hold_cleanup(ds, htag, minor);
+                (void) snprintf(name, sizeof(name), "%llx",
+                    (u_longlong_t)ds->ds_object);
+
+                if (nvlist_lookup_nvlist(dduha->dduha_tmpholds, name, &tags) != 0) {
+                        tags = fnvlist_alloc();
+                        fnvlist_add_boolean(tags, htag);
+                        fnvlist_add_nvlist(dduha->dduha_tmpholds, name, tags);
+                        fnvlist_free(tags);
+                } else {
+                        fnvlist_add_boolean(tags, htag);
+                }
         }
 
         spa_history_log_internal_ds(ds, "hold", tx,
             "tag=%s temp=%d refs=%llu",
             htag, minor != 0, ds->ds_userrefs);
 }
 
+typedef struct zfs_hold_cleanup_arg {
+        char zhca_spaname[MAXNAMELEN];
+        uint64_t zhca_spa_load_guid;
+        nvlist_t *zhca_holds;
+} zfs_hold_cleanup_arg_t;
+
+static void
+dsl_dataset_user_release_onexit(void *arg)
+{
+        zfs_hold_cleanup_arg_t *ca = (zfs_hold_cleanup_arg_t *)arg;
+        spa_t *spa;
+        int error;
+
+        error = spa_open(ca->zhca_spaname, &spa, FTAG);
+        if (error != 0) {
+                zfs_dbgmsg("couldn't release holds on pool=%s "
+                    "because pool is no longer loaded",
+                    ca->zhca_spaname);
+                return;
+        }
+        if (spa_load_guid(spa) != ca->zhca_spa_load_guid) {
+                zfs_dbgmsg("couldn't release holds on pool=%s "
+                    "because pool is no longer loaded (guid doesn't match)",
+                    ca->zhca_spaname);
+                spa_close(spa, FTAG);
+                return;
+        }
+
+        (void) dsl_dataset_user_release_tmp(spa_get_dsl(spa), ca->zhca_holds);
+        fnvlist_free(ca->zhca_holds);
+        kmem_free(ca, sizeof(zfs_hold_cleanup_arg_t));
+        spa_close(spa, FTAG);
+}
+
+static void
+dsl_register_onexit_hold_cleanup(spa_t *spa, nvlist_t *holds, minor_t minor)
+{
+        zfs_hold_cleanup_arg_t *ca;
+
+        if (minor == 0 || nvlist_next_nvpair(holds, NULL) == NULL) {
+                fnvlist_free(holds);
+                return;
+        }
+
+        ASSERT(spa != NULL);
+        ca = kmem_alloc(sizeof (*ca), KM_SLEEP);
+
+        (void) strlcpy(ca->zhca_spaname, spa_name(spa),
+            sizeof (ca->zhca_spaname));
+        ca->zhca_spa_load_guid = spa_load_guid(spa);
+        ca->zhca_holds = holds;
+        VERIFY0(zfs_onexit_add_cb(minor,
+            dsl_dataset_user_release_onexit, ca, NULL));
+}
+
+void
+dsl_dataset_user_hold_sync_one(dsl_dataset_t *ds, const char *htag,
+    minor_t minor, uint64_t now, dmu_tx_t *tx)
+{
+        dsl_dataset_user_hold_arg_t dduha;
+
+        dduha.dduha_spa = NULL;
+        dduha.dduha_holds = NULL;
+        dduha.dduha_tmpholds = fnvlist_alloc();
+        dduha.dduha_errlist = NULL;
+        dduha.dduha_minor = minor;
+        dduha.dduha_holds_created = B_FALSE;
+
+        dsl_dataset_user_hold_sync_one_impl(&dduha, ds, htag, minor, now, tx);
+        dsl_register_onexit_hold_cleanup(dsl_dataset_get_spa(ds),
+            dduha.dduha_tmpholds, minor);
+}
+
 static void
 dsl_dataset_user_hold_sync(void *arg, dmu_tx_t *tx)
 {
         dsl_dataset_user_hold_arg_t *dduha = arg;
         dsl_pool_t *dp = dmu_tx_pool(tx);

@@ -164,75 +283,145 @@
         uint64_t now = gethrestime_sec();
 
         for (pair = nvlist_next_nvpair(dduha->dduha_holds, NULL); pair != NULL;
             pair = nvlist_next_nvpair(dduha->dduha_holds, pair)) {
                 dsl_dataset_t *ds;
-                VERIFY0(dsl_dataset_hold(dp, nvpair_name(pair), FTAG, &ds));
-                dsl_dataset_user_hold_sync_one(ds, fnvpair_value_string(pair),
-                    dduha->dduha_minor, now, tx);
+                char *name;
+                int error;
+
+                name = nvpair_name(pair);
+                error = dsl_dataset_hold(dp, name, FTAG, &ds);
+                if (error == 0) {
+                        dsl_dataset_user_hold_sync_one_impl(dduha, ds,
+                            fnvpair_value_string(pair), dduha->dduha_minor,
+                            now, tx);
                 dsl_dataset_rele(ds, FTAG);
+                        dduha->dduha_holds_created = B_TRUE;
+                } else if (dduha->dduha_errlist != NULL) {
+                        /*
+                         * We register ENOENT errors so they can be correctly
+                         * reported if needed, such as when all holds fail.
+                         */
+                        fnvlist_add_int32(dduha->dduha_errlist, name, error);
         }
+        }
+        dduha->dduha_spa = dp->dp_spa;
 }
 
 /*
+ * The full semantics of this function are described in the comment above
+ * lzc_hold().
+ *
+ * To summarize:
  * holds is nvl of snapname -> holdname
  * errlist will be filled in with snapname -> error
- * if cleanup_minor is not 0, the holds will be temporary, cleaned up
- * when the process exits.
  *
- * if any fails, all will fail.
+ * The snaphosts must all be in the same pool.
+ *
+ * Holds for snapshots that don't exist will be skipped.
+ *
+ * If none of the snapshots for requested holds exist then ENOENT will be
+ * returned.
+ *
+ * If cleanup_minor is not 0, the holds will be temporary, which will be cleaned
+ * up when the process exits.
+ *
+ * On success all the holds, for snapshots that existed, will be created and 0
+ * will be returned.
+ *
+ * On failure no holds will be created, the errlist will be filled in,
+ * and an errno will returned.
+ *
+ * In all cases the errlist will contain entries for holds where the snapshot
+ * didn't exist.
  */
 int
 dsl_dataset_user_hold(nvlist_t *holds, minor_t cleanup_minor, nvlist_t *errlist)
 {
         dsl_dataset_user_hold_arg_t dduha;
         nvpair_t *pair;
+        int ret;
 
         pair = nvlist_next_nvpair(holds, NULL);
         if (pair == NULL)
                 return (0);
 
+        dduha.dduha_spa = NULL;
         dduha.dduha_holds = holds;
+        dduha.dduha_tmpholds = fnvlist_alloc();
         dduha.dduha_errlist = errlist;
         dduha.dduha_minor = cleanup_minor;
+        dduha.dduha_holds_created = B_FALSE;
 
-        return (dsl_sync_task(nvpair_name(pair), dsl_dataset_user_hold_check,
-            dsl_dataset_user_hold_sync, &dduha, fnvlist_num_pairs(holds)));
+        ret = dsl_sync_task(nvpair_name(pair), dsl_dataset_user_hold_check,
+            dsl_dataset_user_hold_sync, &dduha, fnvlist_num_pairs(holds));
+        if (ret == 0) {
+                /* Check we created at least one hold. */
+                if (dduha.dduha_holds_created) {
+                        dsl_register_onexit_hold_cleanup(dduha.dduha_spa,
+                            dduha.dduha_tmpholds, cleanup_minor);
+                } else {
+                        fnvlist_free(dduha.dduha_tmpholds);
+                        ret = ENOENT;
+                }
+        } else {
+                fnvlist_free(dduha.dduha_tmpholds);
+        }
+
+        return (ret);
 }
 
+typedef int (dsl_holdfunc_t)(dsl_pool_t *dp, const char *name, void *tag,
+    dsl_dataset_t **dsp);
+
 typedef struct dsl_dataset_user_release_arg {
+        dsl_holdfunc_t *ddura_holdfunc;
         nvlist_t *ddura_holds;
         nvlist_t *ddura_todelete;
         nvlist_t *ddura_errlist;
+        boolean_t ddura_holds_found;
 } dsl_dataset_user_release_arg_t;
 
+/* Place a dataset hold on the snapshot identified by passed dsobj string */
+static
+int dsl_dataset_hold_byobj(dsl_pool_t *dp, const char *dsobj, void *tag,
+    dsl_dataset_t **dsp)
+{
+        return dsl_dataset_hold_obj(dp, strtonum(dsobj, NULL), tag, dsp);
+}
+
 static int
-dsl_dataset_user_release_check_one(dsl_dataset_t *ds,
-    nvlist_t *holds, boolean_t *todelete)
+dsl_dataset_user_release_check_one(dsl_dataset_user_release_arg_t *ddura,
+    dsl_dataset_t *ds, nvlist_t *holds, boolean_t *todelete)
 {
         uint64_t zapobj;
         nvpair_t *pair;
         objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
         int error;
         int numholds = 0;
+        int ret;
 
         *todelete = B_FALSE;
+        ret = 0;
 
         if (!dsl_dataset_is_snapshot(ds))
                 return (SET_ERROR(EINVAL));
 
         zapobj = ds->ds_phys->ds_userrefs_obj;
         if (zapobj == 0)
                 return (SET_ERROR(ESRCH));
 
         for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
             pair = nvlist_next_nvpair(holds, pair)) {
-                /* Make sure the hold exists */
                 uint64_t tmp;
+
                 error = zap_lookup(mos, zapobj, nvpair_name(pair), 8, 1, &tmp);
-                if (error == ENOENT)
-                        error = SET_ERROR(ESRCH);
+                /* Non-existent holds aren't always fatal. */
+                if (error == ENOENT) {
+                        ret = error;
+                        continue;
+                }
                 if (error != 0)
                         return (error);
                 numholds++;
         }
 

@@ -242,17 +431,22 @@
 
                 if (dsl_dataset_long_held(ds))
                         return (SET_ERROR(EBUSY));
                 *todelete = B_TRUE;
         }
-        return (0);
+
+        if (numholds != 0)
+                ddura->ddura_holds_found = B_TRUE;
+
+        return (ret);
 }
 
 static int
 dsl_dataset_user_release_check(void *arg, dmu_tx_t *tx)
 {
         dsl_dataset_user_release_arg_t *ddura = arg;
+        dsl_holdfunc_t *holdfunc = ddura->ddura_holdfunc;
         dsl_pool_t *dp = dmu_tx_pool(tx);
         nvpair_t *pair;
         int rv = 0;
 
         if (!dmu_tx_is_syncing(tx))

@@ -267,50 +461,80 @@
 
                 error = nvpair_value_nvlist(pair, &holds);
                 if (error != 0)
                         return (SET_ERROR(EINVAL));
 
-                error = dsl_dataset_hold(dp, name, FTAG, &ds);
+                error = holdfunc(dp, name, FTAG, &ds);
                 if (error == 0) {
                         boolean_t deleteme;
-                        error = dsl_dataset_user_release_check_one(ds,
+                        error = dsl_dataset_user_release_check_one(ddura, ds,
                             holds, &deleteme);
-                        if (error == 0 && deleteme) {
+                        /*
+                         * Don't check for error == 0 as deleteme is only set
+                         * to B_TRUE if it's correct to do so dispite the error
+                         * e.g. ENOENT.
+                         */
+                        if (deleteme) {
                                 fnvlist_add_boolean(ddura->ddura_todelete,
                                     name);
                         }
                         dsl_dataset_rele(ds, FTAG);
                 }
                 if (error != 0) {
                         if (ddura->ddura_errlist != NULL) {
                                 fnvlist_add_int32(ddura->ddura_errlist,
                                     name, error);
                         }
+                        /* Non-existent holds aren't always fatal. */
+                        if (error != ENOENT)
                         rv = error;
                 }
         }
+
+        /*
+         * None of the specified holds existed so avoid the overhead of a sync
+         * and return ENOENT.
+         */
+        if (rv == 0 && !ddura->ddura_holds_found)
+                rv = ENOENT;
+
         return (rv);
 }
 
 static void
-dsl_dataset_user_release_sync_one(dsl_dataset_t *ds, nvlist_t *holds,
-    dmu_tx_t *tx)
+dsl_dataset_user_release_sync_one(dsl_dataset_user_release_arg_t *ddura,
+    dsl_dataset_t *ds, nvlist_t *holds, dmu_tx_t *tx)
 {
         dsl_pool_t *dp = ds->ds_dir->dd_pool;
         objset_t *mos = dp->dp_meta_objset;
-        uint64_t zapobj;
-        int error;
         nvpair_t *pair;
 
         for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
             pair = nvlist_next_nvpair(holds, pair)) {
-                ds->ds_userrefs--;
-                error = dsl_pool_user_release(dp, ds->ds_object,
-                    nvpair_name(pair), tx);
+                uint64_t zapobj;
+                int error;
+                char *name;
+
+                name = nvpair_name(pair);
+
+                /* Remove temporary hold if one exists. */
+                error = dsl_pool_user_release(dp, ds->ds_object, name, tx);
                 VERIFY(error == 0 || error == ENOENT);
+
+                /* Remove user hold if one exists. */
                 zapobj = ds->ds_phys->ds_userrefs_obj;
-                VERIFY0(zap_remove(mos, zapobj, nvpair_name(pair), tx));
+                error = zap_remove(mos, zapobj, name, tx);
+                if (error == ENOENT)
+                        continue;
+                VERIFY0(error);
+
+                /* Only if we removed a hold do we decrement userrefs. */
+                mutex_enter(&ds->ds_lock);
+                ds->ds_userrefs--;
+                mutex_exit(&ds->ds_lock);
+
+                ddura->ddura_holds_found = B_TRUE;
 
                 spa_history_log_internal_ds(ds, "release", tx,
                     "tag=%s refs=%lld", nvpair_name(pair),
                     (longlong_t)ds->ds_userrefs);
         }

@@ -318,187 +542,151 @@
 
 static void
 dsl_dataset_user_release_sync(void *arg, dmu_tx_t *tx)
 {
         dsl_dataset_user_release_arg_t *ddura = arg;
+        dsl_holdfunc_t *holdfunc = ddura->ddura_holdfunc;
         dsl_pool_t *dp = dmu_tx_pool(tx);
         nvpair_t *pair;
 
+        /*
+         * Even though check suggested that at least one of our holds where
+         * found this may have changed. Recalculate ddura_holds_found so that
+         * we can return ENOENT from the caller in the case that no holds
+         * where actually released.
+         */
+        ddura->ddura_holds_found = B_FALSE;
+
         for (pair = nvlist_next_nvpair(ddura->ddura_holds, NULL); pair != NULL;
             pair = nvlist_next_nvpair(ddura->ddura_holds, pair)) {
                 dsl_dataset_t *ds;
+                int error;
 
-                VERIFY0(dsl_dataset_hold(dp, nvpair_name(pair), FTAG, &ds));
-                dsl_dataset_user_release_sync_one(ds,
+                error = holdfunc(dp, nvpair_name(pair), FTAG, &ds);
+                if (error == ENOENT)
+                        continue;
+                VERIFY0(error);
+
+                dsl_dataset_user_release_sync_one(ddura, ds,
                     fnvpair_value_nvlist(pair), tx);
-                if (nvlist_exists(ddura->ddura_todelete,
-                    nvpair_name(pair))) {
+                if (nvlist_exists(ddura->ddura_todelete, nvpair_name(pair))) {
                         ASSERT(ds->ds_userrefs == 0 &&
                             ds->ds_phys->ds_num_children == 1 &&
                             DS_IS_DEFER_DESTROY(ds));
                         dsl_destroy_snapshot_sync_impl(ds, B_FALSE, tx);
                 }
                 dsl_dataset_rele(ds, FTAG);
         }
 }
 
 /*
+ * The full semantics of this function are described in the comment above
+ * lzc_release().
+ *
+ * To summarize:
+ * Releases holds specified in the nvl holds.
+ *
  * holds is nvl of snapname -> { holdname, ... }
  * errlist will be filled in with snapname -> error
  *
- * if any fails, all will fail.
+ * If tmpdp is not NULL the names for holds should be the dbobj's of snapshots,
+ * otherwise they should be the names of shapshots.
+ *
+ * As a release may cause snapshots to be destroyed this trys to ensure they
+ * aren't mounted.
+ *
+ * The release of non-existent holds are skipped.
+ *
+ * At least one hold must have been released for the this function to succeed
+ * and return 0.
  */
-int
-dsl_dataset_user_release(nvlist_t *holds, nvlist_t *errlist)
+static int
+dsl_dataset_user_release_impl(nvlist_t *holds, nvlist_t *errlist,
+    dsl_pool_t *tmpdp)
 {
         dsl_dataset_user_release_arg_t ddura;
         nvpair_t *pair;
+        char *pool;
         int error;
 
         pair = nvlist_next_nvpair(holds, NULL);
         if (pair == NULL)
                 return (0);
 
-        ddura.ddura_holds = holds;
-        ddura.ddura_errlist = errlist;
-        ddura.ddura_todelete = fnvlist_alloc();
-
-        error = dsl_sync_task(nvpair_name(pair), dsl_dataset_user_release_check,
-            dsl_dataset_user_release_sync, &ddura, fnvlist_num_pairs(holds));
-        fnvlist_free(ddura.ddura_todelete);
-        return (error);
-}
-
-typedef struct dsl_dataset_user_release_tmp_arg {
-        uint64_t ddurta_dsobj;
-        nvlist_t *ddurta_holds;
-        boolean_t ddurta_deleteme;
-} dsl_dataset_user_release_tmp_arg_t;
-
-static int
-dsl_dataset_user_release_tmp_check(void *arg, dmu_tx_t *tx)
-{
-        dsl_dataset_user_release_tmp_arg_t *ddurta = arg;
-        dsl_pool_t *dp = dmu_tx_pool(tx);
-        dsl_dataset_t *ds;
-        int error;
-
-        if (!dmu_tx_is_syncing(tx))
-                return (0);
-
-        error = dsl_dataset_hold_obj(dp, ddurta->ddurta_dsobj, FTAG, &ds);
-        if (error)
-                return (error);
-
-        error = dsl_dataset_user_release_check_one(ds,
-            ddurta->ddurta_holds, &ddurta->ddurta_deleteme);
-        dsl_dataset_rele(ds, FTAG);
-        return (error);
-}
-
-static void
-dsl_dataset_user_release_tmp_sync(void *arg, dmu_tx_t *tx)
-{
-        dsl_dataset_user_release_tmp_arg_t *ddurta = arg;
-        dsl_pool_t *dp = dmu_tx_pool(tx);
-        dsl_dataset_t *ds;
-
-        VERIFY0(dsl_dataset_hold_obj(dp, ddurta->ddurta_dsobj, FTAG, &ds));
-        dsl_dataset_user_release_sync_one(ds, ddurta->ddurta_holds, tx);
-        if (ddurta->ddurta_deleteme) {
-                ASSERT(ds->ds_userrefs == 0 &&
-                    ds->ds_phys->ds_num_children == 1 &&
-                    DS_IS_DEFER_DESTROY(ds));
-                dsl_destroy_snapshot_sync_impl(ds, B_FALSE, tx);
-        }
-        dsl_dataset_rele(ds, FTAG);
-}
-
-/*
- * Called at spa_load time to release a stale temporary user hold.
- * Also called by the onexit code.
+#ifdef _KERNEL
+        /*
+         * The release may cause snapshots to be destroyed; make sure they
+         * are not mounted.
  */
-void
-dsl_dataset_user_release_tmp(dsl_pool_t *dp, uint64_t dsobj, const char *htag)
-{
-        dsl_dataset_user_release_tmp_arg_t ddurta;
+        if (tmpdp != NULL) {
+                /* Temporary holds are specified by dbobj. */
+                ddura.ddura_holdfunc = dsl_dataset_hold_byobj;
+                pool = spa_name(tmpdp->dp_spa);
+
+                dsl_pool_config_enter(tmpdp, FTAG);
+                for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
+                    pair = nvlist_next_nvpair(holds, pair)) {
         dsl_dataset_t *ds;
-        int error;
 
-#ifdef _KERNEL
-        /* Make sure it is not mounted. */
-        dsl_pool_config_enter(dp, FTAG);
-        error = dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds);
+                        error = dsl_dataset_hold_byobj(tmpdp, nvpair_name(pair),
+                            FTAG, &ds);
         if (error == 0) {
                 char name[MAXNAMELEN];
                 dsl_dataset_name(ds, name);
                 dsl_dataset_rele(ds, FTAG);
-                dsl_pool_config_exit(dp, FTAG);
                 zfs_unmount_snap(name);
+                        }
+                }
+                dsl_pool_config_exit(tmpdp, FTAG);
         } else {
-                dsl_pool_config_exit(dp, FTAG);
+                /* Non-temporary holds are specified by name. */
+                ddura.ddura_holdfunc = dsl_dataset_hold;
+                pool = nvpair_name(pair);
+
+                for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL;
+                    pair = nvlist_next_nvpair(holds, pair))
+                        zfs_unmount_snap(nvpair_name(pair));
         }
 #endif
 
-        ddurta.ddurta_dsobj = dsobj;
-        ddurta.ddurta_holds = fnvlist_alloc();
-        fnvlist_add_boolean(ddurta.ddurta_holds, htag);
-
-        (void) dsl_sync_task(spa_name(dp->dp_spa),
-            dsl_dataset_user_release_tmp_check,
-            dsl_dataset_user_release_tmp_sync, &ddurta, 1);
-        fnvlist_free(ddurta.ddurta_holds);
-}
+        ddura.ddura_holds = holds;
+        ddura.ddura_errlist = errlist;
+        ddura.ddura_todelete = fnvlist_alloc();
+        ddura.ddura_holds_found = B_FALSE;
 
-typedef struct zfs_hold_cleanup_arg {
-        char zhca_spaname[MAXNAMELEN];
-        uint64_t zhca_spa_load_guid;
-        uint64_t zhca_dsobj;
-        char zhca_htag[MAXNAMELEN];
-} zfs_hold_cleanup_arg_t;
+        error = dsl_sync_task(pool, dsl_dataset_user_release_check,
+            dsl_dataset_user_release_sync, &ddura,
+            fnvlist_num_pairs(holds));
+        fnvlist_free(ddura.ddura_todelete);
 
-static void
-dsl_dataset_user_release_onexit(void *arg)
-{
-        zfs_hold_cleanup_arg_t *ca = arg;
-        spa_t *spa;
-        int error;
+        /* If at least one hold wasn't removed return ENOENT. */
+        if (error == 0 && !ddura.ddura_holds_found)
+                error = ENOENT;
 
-        error = spa_open(ca->zhca_spaname, &spa, FTAG);
-        if (error != 0) {
-                zfs_dbgmsg("couldn't release hold on pool=%s ds=%llu tag=%s "
-                    "because pool is no longer loaded",
-                    ca->zhca_spaname, ca->zhca_dsobj, ca->zhca_htag);
-                return;
-        }
-        if (spa_load_guid(spa) != ca->zhca_spa_load_guid) {
-                zfs_dbgmsg("couldn't release hold on pool=%s ds=%llu tag=%s "
-                    "because pool is no longer loaded (guid doesn't match)",
-                    ca->zhca_spaname, ca->zhca_dsobj, ca->zhca_htag);
-                spa_close(spa, FTAG);
-                return;
-        }
+        return (error);
+}
 
-        dsl_dataset_user_release_tmp(spa_get_dsl(spa),
-            ca->zhca_dsobj, ca->zhca_htag);
-        kmem_free(ca, sizeof (zfs_hold_cleanup_arg_t));
-        spa_close(spa, FTAG);
+/*
+ * holds is nvl of snapname -> { holdname, ... }
+ * errlist will be filled in with snapname -> error
+ *
+ * if any fails, all will fail.
+ */
+int
+dsl_dataset_user_release(nvlist_t *holds, nvlist_t *errlist)
+{
+        return dsl_dataset_user_release_impl(holds, errlist, NULL);
 }
 
+/*
+ * holds is nvl of snapdsobj -> { holdname, ... }
+ */
 void
-dsl_register_onexit_hold_cleanup(dsl_dataset_t *ds, const char *htag,
-    minor_t minor)
+dsl_dataset_user_release_tmp(struct dsl_pool *dp, nvlist_t *holds)
 {
-        zfs_hold_cleanup_arg_t *ca = kmem_alloc(sizeof (*ca), KM_SLEEP);
-        spa_t *spa = dsl_dataset_get_spa(ds);
-        (void) strlcpy(ca->zhca_spaname, spa_name(spa),
-            sizeof (ca->zhca_spaname));
-        ca->zhca_spa_load_guid = spa_load_guid(spa);
-        ca->zhca_dsobj = ds->ds_object;
-        (void) strlcpy(ca->zhca_htag, htag, sizeof (ca->zhca_htag));
-        VERIFY0(zfs_onexit_add_cb(minor,
-            dsl_dataset_user_release_onexit, ca, NULL));
+        ASSERT(dp != NULL);
+        (void) dsl_dataset_user_release_impl(holds, NULL, dp);
 }
 
 int
 dsl_dataset_get_holds(const char *dsname, nvlist_t *nvl)
 {