Print this page
3740 Poor ZFS send / receive performance due to snapshot hold / release processing
Submitted by: Steven Hartland <steven.hartland@multiplay.co.uk>


   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2012 by Delphix. All rights reserved.
  25  * Copyright (c) 2012 DEY Storage Systems, Inc.  All rights reserved.
  26  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  27  * Copyright (c) 2013 Martin Matuska. All rights reserved.

  28  */
  29 
  30 #include <ctype.h>
  31 #include <errno.h>
  32 #include <libintl.h>
  33 #include <math.h>
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <unistd.h>
  38 #include <stddef.h>
  39 #include <zone.h>
  40 #include <fcntl.h>
  41 #include <sys/mntent.h>
  42 #include <sys/mount.h>
  43 #include <priv.h>
  44 #include <pwd.h>
  45 #include <grp.h>
  46 #include <stddef.h>
  47 #include <ucred.h>


3087             errno != ENOENT) {
3088                 return (zfs_standard_error_fmt(zhp->zfs_hdl, errno,
3089                     dgettext(TEXT_DOMAIN, "cannot destroy '%s'"),
3090                     zhp->zfs_name));
3091         }
3092 
3093         remove_mountpoint(zhp);
3094 
3095         return (0);
3096 }
3097 
3098 struct destroydata {
3099         nvlist_t *nvl;
3100         const char *snapname;
3101 };
3102 
3103 static int
3104 zfs_check_snap_cb(zfs_handle_t *zhp, void *arg)
3105 {
3106         struct destroydata *dd = arg;
3107         zfs_handle_t *szhp;
3108         char name[ZFS_MAXNAMELEN];
3109         int rv = 0;
3110 
3111         (void) snprintf(name, sizeof (name),
3112             "%s@%s", zhp->zfs_name, dd->snapname);
3113 
3114         szhp = make_dataset_handle(zhp->zfs_hdl, name);
3115         if (szhp) {
3116                 verify(nvlist_add_boolean(dd->nvl, name) == 0);
3117                 zfs_close(szhp);
3118         }
3119 
3120         rv = zfs_iter_filesystems(zhp, zfs_check_snap_cb, dd);
3121         zfs_close(zhp);
3122         return (rv);
3123 }
3124 
3125 /*
3126  * Destroys all snapshots with the given name in zhp & descendants.
3127  */
3128 int
3129 zfs_destroy_snaps(zfs_handle_t *zhp, char *snapname, boolean_t defer)
3130 {
3131         int ret;
3132         struct destroydata dd = { 0 };
3133 
3134         dd.snapname = snapname;
3135         verify(nvlist_alloc(&dd.nvl, NV_UNIQUE_NAME, 0) == 0);
3136         (void) zfs_check_snap_cb(zfs_handle_dup(zhp), &dd);
3137 
3138         if (nvlist_next_nvpair(dd.nvl, NULL) == NULL) {
3139                 ret = zfs_standard_error_fmt(zhp->zfs_hdl, ENOENT,
3140                     dgettext(TEXT_DOMAIN, "cannot destroy '%s@%s'"),
3141                     zhp->zfs_name, snapname);
3142         } else {
3143                 ret = zfs_destroy_snaps_nvl(zhp->zfs_hdl, dd.nvl, defer);
3144         }
3145         nvlist_free(dd.nvl);
3146         return (ret);
3147 }
3148 
3149 /*
3150  * Destroys all the snapshots named in the nvlist.
3151  */
3152 int
3153 zfs_destroy_snaps_nvl(libzfs_handle_t *hdl, nvlist_t *snaps, boolean_t defer)
3154 {
3155         int ret;
3156         nvlist_t *errlist;
3157 
3158         ret = lzc_destroy_snaps(snaps, defer, &errlist);
3159 
3160         if (ret == 0)
3161                 return (0);
3162 
3163         if (nvlist_next_nvpair(errlist, NULL) == NULL) {
3164                 char errbuf[1024];
3165                 (void) snprintf(errbuf, sizeof (errbuf),
3166                     dgettext(TEXT_DOMAIN, "cannot destroy snapshots"));
3167 
3168                 ret = zfs_standard_error(hdl, ret, errbuf);
3169         }
3170         for (nvpair_t *pair = nvlist_next_nvpair(errlist, NULL);
3171             pair != NULL; pair = nvlist_next_nvpair(errlist, pair)) {
3172                 char errbuf[1024];
3173                 (void) snprintf(errbuf, sizeof (errbuf),
3174                     dgettext(TEXT_DOMAIN, "cannot destroy snapshot %s"),
3175                     nvpair_name(pair));
3176 
3177                 switch (fnvpair_value_int32(pair)) {
3178                 case EEXIST:
3179                         zfs_error_aux(hdl,
3180                             dgettext(TEXT_DOMAIN, "snapshot is cloned"));
3181                         ret = zfs_error(hdl, EZFS_EXISTS, errbuf);
3182                         break;
3183                 default:


4065                                 return (ret);
4066                         zua++;
4067                         zc.zc_nvlist_dst_size -= sizeof (zfs_useracct_t);
4068                 }
4069         }
4070 
4071         return (0);
4072 }
4073 
4074 struct holdarg {
4075         nvlist_t *nvl;
4076         const char *snapname;
4077         const char *tag;
4078         boolean_t recursive;
4079 };
4080 
4081 static int
4082 zfs_hold_one(zfs_handle_t *zhp, void *arg)
4083 {
4084         struct holdarg *ha = arg;
4085         zfs_handle_t *szhp;
4086         char name[ZFS_MAXNAMELEN];
4087         int rv = 0;
4088 
4089         (void) snprintf(name, sizeof (name),
4090             "%s@%s", zhp->zfs_name, ha->snapname);
4091 
4092         szhp = make_dataset_handle(zhp->zfs_hdl, name);
4093         if (szhp) {
4094                 fnvlist_add_string(ha->nvl, name, ha->tag);
4095                 zfs_close(szhp);
4096         }
4097 
4098         if (ha->recursive)
4099                 rv = zfs_iter_filesystems(zhp, zfs_hold_one, ha);
4100         zfs_close(zhp);
4101         return (rv);
4102 }
4103 
4104 int
4105 zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
4106     boolean_t recursive, boolean_t enoent_ok, int cleanup_fd)
4107 {
4108         int ret;
4109         struct holdarg ha;
4110         nvlist_t *errors;
4111         libzfs_handle_t *hdl = zhp->zfs_hdl;
4112         char errbuf[1024];
4113         nvpair_t *elem;
4114 
4115         ha.nvl = fnvlist_alloc();
4116         ha.snapname = snapname;
4117         ha.tag = tag;
4118         ha.recursive = recursive;
4119         (void) zfs_hold_one(zfs_handle_dup(zhp), &ha);
4120 
4121         if (nvlist_next_nvpair(ha.nvl, NULL) == NULL) {


4122                 fnvlist_free(ha.nvl);
4123                 ret = ENOENT;
4124                 if (!enoent_ok) {
4125                         (void) snprintf(errbuf, sizeof (errbuf),
4126                             dgettext(TEXT_DOMAIN,
4127                             "cannot hold snapshot '%s@%s'"),
4128                             zhp->zfs_name, snapname);
4129                         (void) zfs_standard_error(hdl, ret, errbuf);
4130                 }
4131                 return (ret);
4132         }
4133 
4134         ret = lzc_hold(ha.nvl, cleanup_fd, &errors);
4135         fnvlist_free(ha.nvl);
4136 
4137         if (ret == 0)

















4138                 return (0);

4139 
4140         if (nvlist_next_nvpair(errors, NULL) == NULL) {
4141                 /* no hold-specific errors */
4142                 (void) snprintf(errbuf, sizeof (errbuf),
4143                     dgettext(TEXT_DOMAIN, "cannot hold"));
4144                 switch (ret) {
4145                 case ENOTSUP:
4146                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4147                             "pool must be upgraded"));
4148                         (void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
4149                         break;
4150                 case EINVAL:
4151                         (void) zfs_error(hdl, EZFS_BADTYPE, errbuf);
4152                         break;
4153                 default:
4154                         (void) zfs_standard_error(hdl, ret, errbuf);
4155                 }
4156         }
4157 
4158         for (elem = nvlist_next_nvpair(errors, NULL);
4159             elem != NULL;
4160             elem = nvlist_next_nvpair(errors, elem)) {
4161                 (void) snprintf(errbuf, sizeof (errbuf),
4162                     dgettext(TEXT_DOMAIN,
4163                     "cannot hold snapshot '%s'"), nvpair_name(elem));
4164                 switch (fnvpair_value_int32(elem)) {
4165                 case E2BIG:
4166                         /*
4167                          * Temporary tags wind up having the ds object id
4168                          * prepended. So even if we passed the length check
4169                          * above, it's still possible for the tag to wind
4170                          * up being slightly too long.
4171                          */
4172                         (void) zfs_error(hdl, EZFS_TAGTOOLONG, errbuf);
4173                         break;
4174                 case EINVAL:
4175                         (void) zfs_error(hdl, EZFS_BADTYPE, errbuf);
4176                         break;
4177                 case EEXIST:
4178                         (void) zfs_error(hdl, EZFS_REFTAG_HOLD, errbuf);
4179                         break;
4180                 case ENOENT:
4181                         if (enoent_ok)
4182                                 return (ENOENT);
4183                         /* FALLTHROUGH */
4184                 default:
4185                         (void) zfs_standard_error(hdl,
4186                             fnvpair_value_int32(elem), errbuf);
4187                 }
4188         }
4189 
4190         fnvlist_free(errors);
4191         return (ret);
4192 }
4193 
4194 struct releasearg {
4195         nvlist_t *nvl;
4196         const char *snapname;
4197         const char *tag;
4198         boolean_t recursive;
4199 };
4200 
4201 static int
4202 zfs_release_one(zfs_handle_t *zhp, void *arg)
4203 {
4204         struct holdarg *ha = arg;
4205         zfs_handle_t *szhp;
4206         char name[ZFS_MAXNAMELEN];
4207         int rv = 0;
4208 
4209         (void) snprintf(name, sizeof (name),
4210             "%s@%s", zhp->zfs_name, ha->snapname);
4211 
4212         szhp = make_dataset_handle(zhp->zfs_hdl, name);
4213         if (szhp) {
4214                 nvlist_t *holds = fnvlist_alloc();
4215                 fnvlist_add_boolean(holds, ha->tag);
4216                 fnvlist_add_nvlist(ha->nvl, name, holds);
4217                 zfs_close(szhp);
4218         }
4219 
4220         if (ha->recursive)
4221                 rv = zfs_iter_filesystems(zhp, zfs_release_one, ha);
4222         zfs_close(zhp);
4223         return (rv);
4224 }
4225 
4226 int
4227 zfs_release(zfs_handle_t *zhp, const char *snapname, const char *tag,
4228     boolean_t recursive)
4229 {
4230         int ret;
4231         struct holdarg ha;
4232         nvlist_t *errors;
4233         nvpair_t *elem;
4234         libzfs_handle_t *hdl = zhp->zfs_hdl;
4235         char errbuf[1024];
4236 
4237         ha.nvl = fnvlist_alloc();
4238         ha.snapname = snapname;
4239         ha.tag = tag;
4240         ha.recursive = recursive;
4241         (void) zfs_release_one(zfs_handle_dup(zhp), &ha);
4242 
4243         if (nvlist_next_nvpair(ha.nvl, NULL) == NULL) {
4244                 fnvlist_free(ha.nvl);
4245                 ret = ENOENT;
4246                 (void) snprintf(errbuf, sizeof (errbuf),
4247                     dgettext(TEXT_DOMAIN,
4248                     "cannot release hold from snapshot '%s@%s'"),
4249                     zhp->zfs_name, snapname);
4250                 (void) zfs_standard_error(hdl, ret, errbuf);
4251                 return (ret);
4252         }
4253 
4254         ret = lzc_release(ha.nvl, &errors);
4255         fnvlist_free(ha.nvl);
4256 
4257         if (ret == 0)


4258                 return (0);

4259 
4260         if (nvlist_next_nvpair(errors, NULL) == NULL) {
4261                 /* no hold-specific errors */
4262                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
4263                     "cannot release"));
4264                 switch (errno) {
4265                 case ENOTSUP:
4266                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4267                             "pool must be upgraded"));
4268                         (void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
4269                         break;
4270                 default:
4271                         (void) zfs_standard_error_fmt(hdl, errno, errbuf);
4272                 }
4273         }
4274 
4275         for (elem = nvlist_next_nvpair(errors, NULL);
4276             elem != NULL;
4277             elem = nvlist_next_nvpair(errors, elem)) {
4278                 (void) snprintf(errbuf, sizeof (errbuf),
4279                     dgettext(TEXT_DOMAIN,
4280                     "cannot release hold from snapshot '%s'"),




   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2012 by Delphix. All rights reserved.
  25  * Copyright (c) 2012 DEY Storage Systems, Inc.  All rights reserved.
  26  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
  27  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  28  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  29  */
  30 
  31 #include <ctype.h>
  32 #include <errno.h>
  33 #include <libintl.h>
  34 #include <math.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <strings.h>
  38 #include <unistd.h>
  39 #include <stddef.h>
  40 #include <zone.h>
  41 #include <fcntl.h>
  42 #include <sys/mntent.h>
  43 #include <sys/mount.h>
  44 #include <priv.h>
  45 #include <pwd.h>
  46 #include <grp.h>
  47 #include <stddef.h>
  48 #include <ucred.h>


3088             errno != ENOENT) {
3089                 return (zfs_standard_error_fmt(zhp->zfs_hdl, errno,
3090                     dgettext(TEXT_DOMAIN, "cannot destroy '%s'"),
3091                     zhp->zfs_name));
3092         }
3093 
3094         remove_mountpoint(zhp);
3095 
3096         return (0);
3097 }
3098 
3099 struct destroydata {
3100         nvlist_t *nvl;
3101         const char *snapname;
3102 };
3103 
3104 static int
3105 zfs_check_snap_cb(zfs_handle_t *zhp, void *arg)
3106 {
3107         struct destroydata *dd = arg;

3108         char name[ZFS_MAXNAMELEN];
3109         int rv = 0;
3110 
3111         (void) snprintf(name, sizeof (name),
3112             "%s@%s", zhp->zfs_name, dd->snapname);
3113 
3114         if (lzc_exists(name))

3115                 verify(nvlist_add_boolean(dd->nvl, name) == 0);


3116 
3117         rv = zfs_iter_filesystems(zhp, zfs_check_snap_cb, dd);
3118         zfs_close(zhp);
3119         return (rv);
3120 }
3121 
3122 /*
3123  * Destroys all snapshots with the given name in zhp & descendants.
3124  */
3125 int
3126 zfs_destroy_snaps(zfs_handle_t *zhp, char *snapname, boolean_t defer)
3127 {
3128         int ret;
3129         struct destroydata dd = { 0 };
3130 
3131         dd.snapname = snapname;
3132         verify(nvlist_alloc(&dd.nvl, NV_UNIQUE_NAME, 0) == 0);
3133         (void) zfs_check_snap_cb(zfs_handle_dup(zhp), &dd);
3134 
3135         if (nvlist_empty(dd.nvl)) {
3136                 ret = zfs_standard_error_fmt(zhp->zfs_hdl, ENOENT,
3137                     dgettext(TEXT_DOMAIN, "cannot destroy '%s@%s'"),
3138                     zhp->zfs_name, snapname);
3139         } else {
3140                 ret = zfs_destroy_snaps_nvl(zhp->zfs_hdl, dd.nvl, defer);
3141         }
3142         nvlist_free(dd.nvl);
3143         return (ret);
3144 }
3145 
3146 /*
3147  * Destroys all the snapshots named in the nvlist.
3148  */
3149 int
3150 zfs_destroy_snaps_nvl(libzfs_handle_t *hdl, nvlist_t *snaps, boolean_t defer)
3151 {
3152         int ret;
3153         nvlist_t *errlist;
3154 
3155         ret = lzc_destroy_snaps(snaps, defer, &errlist);
3156 
3157         if (ret == 0)
3158                 return (0);
3159 
3160         if (nvlist_empty(errlist)) {
3161                 char errbuf[1024];
3162                 (void) snprintf(errbuf, sizeof (errbuf),
3163                     dgettext(TEXT_DOMAIN, "cannot destroy snapshots"));
3164 
3165                 ret = zfs_standard_error(hdl, ret, errbuf);
3166         }
3167         for (nvpair_t *pair = nvlist_next_nvpair(errlist, NULL);
3168             pair != NULL; pair = nvlist_next_nvpair(errlist, pair)) {
3169                 char errbuf[1024];
3170                 (void) snprintf(errbuf, sizeof (errbuf),
3171                     dgettext(TEXT_DOMAIN, "cannot destroy snapshot %s"),
3172                     nvpair_name(pair));
3173 
3174                 switch (fnvpair_value_int32(pair)) {
3175                 case EEXIST:
3176                         zfs_error_aux(hdl,
3177                             dgettext(TEXT_DOMAIN, "snapshot is cloned"));
3178                         ret = zfs_error(hdl, EZFS_EXISTS, errbuf);
3179                         break;
3180                 default:


4062                                 return (ret);
4063                         zua++;
4064                         zc.zc_nvlist_dst_size -= sizeof (zfs_useracct_t);
4065                 }
4066         }
4067 
4068         return (0);
4069 }
4070 
4071 struct holdarg {
4072         nvlist_t *nvl;
4073         const char *snapname;
4074         const char *tag;
4075         boolean_t recursive;
4076 };
4077 
4078 static int
4079 zfs_hold_one(zfs_handle_t *zhp, void *arg)
4080 {
4081         struct holdarg *ha = arg;

4082         char name[ZFS_MAXNAMELEN];
4083         int rv = 0;
4084 
4085         (void) snprintf(name, sizeof (name),
4086             "%s@%s", zhp->zfs_name, ha->snapname);
4087 
4088         if (lzc_exists(name))

4089                 fnvlist_add_string(ha->nvl, name, ha->tag);


4090 
4091         if (ha->recursive)
4092                 rv = zfs_iter_filesystems(zhp, zfs_hold_one, ha);
4093         zfs_close(zhp);
4094         return (rv);
4095 }
4096 
4097 int
4098 zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
4099     boolean_t recursive, int cleanup_fd)
4100 {
4101         int ret;
4102         struct holdarg ha;




4103 
4104         ha.nvl = fnvlist_alloc();
4105         ha.snapname = snapname;
4106         ha.tag = tag;
4107         ha.recursive = recursive;
4108         (void) zfs_hold_one(zfs_handle_dup(zhp), &ha);
4109 
4110         if (nvlist_empty(ha.nvl)) {
4111                 char errbuf[1024];
4112 
4113                 fnvlist_free(ha.nvl);
4114                 ret = ENOENT;

4115                 (void) snprintf(errbuf, sizeof (errbuf),
4116                     dgettext(TEXT_DOMAIN,
4117                     "cannot hold snapshot '%s@%s'"),
4118                     zhp->zfs_name, snapname);
4119                 (void) zfs_standard_error(zhp->zfs_hdl, ret, errbuf);

4120                 return (ret);
4121         }
4122 
4123         ret = zfs_hold_nvl(zhp, cleanup_fd, ha.nvl);
4124         fnvlist_free(ha.nvl);
4125 
4126         return (ret);
4127 }
4128 
4129 int
4130 zfs_hold_nvl(zfs_handle_t *zhp, int cleanup_fd, nvlist_t *holds)
4131 {
4132         int ret;
4133         nvlist_t *errors;
4134         libzfs_handle_t *hdl = zhp->zfs_hdl;
4135         char errbuf[1024];
4136         nvpair_t *elem;
4137 
4138         errors = NULL;
4139         ret = lzc_hold(holds, cleanup_fd, &errors);
4140 
4141         if (ret == 0) {
4142                 /* There may be errors even in the success case. */
4143                 fnvlist_free(errors);
4144                 return (0);
4145         }
4146 
4147         if (nvlist_empty(errors)) {
4148                 /* no hold-specific errors */
4149                 (void) snprintf(errbuf, sizeof (errbuf),
4150                     dgettext(TEXT_DOMAIN, "cannot hold"));
4151                 switch (ret) {
4152                 case ENOTSUP:
4153                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4154                             "pool must be upgraded"));
4155                         (void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
4156                         break;
4157                 case EINVAL:
4158                         (void) zfs_error(hdl, EZFS_BADTYPE, errbuf);
4159                         break;
4160                 default:
4161                         (void) zfs_standard_error(hdl, ret, errbuf);
4162                 }
4163         }
4164 
4165         for (elem = nvlist_next_nvpair(errors, NULL);
4166             elem != NULL;
4167             elem = nvlist_next_nvpair(errors, elem)) {
4168                 (void) snprintf(errbuf, sizeof (errbuf),
4169                     dgettext(TEXT_DOMAIN,
4170                     "cannot hold snapshot '%s'"), nvpair_name(elem));
4171                 switch (fnvpair_value_int32(elem)) {
4172                 case E2BIG:
4173                         /*
4174                          * Temporary tags wind up having the ds object id
4175                          * prepended. So even if we passed the length check
4176                          * above, it's still possible for the tag to wind
4177                          * up being slightly too long.
4178                          */
4179                         (void) zfs_error(hdl, EZFS_TAGTOOLONG, errbuf);
4180                         break;
4181                 case EINVAL:
4182                         (void) zfs_error(hdl, EZFS_BADTYPE, errbuf);
4183                         break;
4184                 case EEXIST:
4185                         (void) zfs_error(hdl, EZFS_REFTAG_HOLD, errbuf);
4186                         break;




4187                 default:
4188                         (void) zfs_standard_error(hdl,
4189                             fnvpair_value_int32(elem), errbuf);
4190                 }
4191         }
4192 
4193         fnvlist_free(errors);
4194         return (ret);
4195 }
4196 







4197 static int
4198 zfs_release_one(zfs_handle_t *zhp, void *arg)
4199 {
4200         struct holdarg *ha = arg;

4201         char name[ZFS_MAXNAMELEN];
4202         int rv = 0;
4203 
4204         (void) snprintf(name, sizeof (name),
4205             "%s@%s", zhp->zfs_name, ha->snapname);
4206 
4207         if (lzc_exists(name)) {

4208                 nvlist_t *holds = fnvlist_alloc();
4209                 fnvlist_add_boolean(holds, ha->tag);
4210                 fnvlist_add_nvlist(ha->nvl, name, holds);
4211                 fnvlist_free(holds);
4212         }
4213 
4214         if (ha->recursive)
4215                 rv = zfs_iter_filesystems(zhp, zfs_release_one, ha);
4216         zfs_close(zhp);
4217         return (rv);
4218 }
4219 
4220 int
4221 zfs_release(zfs_handle_t *zhp, const char *snapname, const char *tag,
4222     boolean_t recursive)
4223 {
4224         int ret;
4225         struct holdarg ha;
4226         nvlist_t *errors = NULL;
4227         nvpair_t *elem;
4228         libzfs_handle_t *hdl = zhp->zfs_hdl;
4229         char errbuf[1024];
4230 
4231         ha.nvl = fnvlist_alloc();
4232         ha.snapname = snapname;
4233         ha.tag = tag;
4234         ha.recursive = recursive;
4235         (void) zfs_release_one(zfs_handle_dup(zhp), &ha);
4236 
4237         if (nvlist_empty(ha.nvl)) {
4238                 fnvlist_free(ha.nvl);
4239                 ret = ENOENT;
4240                 (void) snprintf(errbuf, sizeof (errbuf),
4241                     dgettext(TEXT_DOMAIN,
4242                     "cannot release hold from snapshot '%s@%s'"),
4243                     zhp->zfs_name, snapname);
4244                 (void) zfs_standard_error(hdl, ret, errbuf);
4245                 return (ret);
4246         }
4247 
4248         ret = lzc_release(ha.nvl, &errors);
4249         fnvlist_free(ha.nvl);
4250 
4251         if (ret == 0) {
4252                 /* There may be errors even in the success case. */
4253                 fnvlist_free(errors);
4254                 return (0);
4255         }
4256 
4257         if (nvlist_empty(errors)) {
4258                 /* no hold-specific errors */
4259                 (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
4260                     "cannot release"));
4261                 switch (errno) {
4262                 case ENOTSUP:
4263                         zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
4264                             "pool must be upgraded"));
4265                         (void) zfs_error(hdl, EZFS_BADVERSION, errbuf);
4266                         break;
4267                 default:
4268                         (void) zfs_standard_error_fmt(hdl, errno, errbuf);
4269                 }
4270         }
4271 
4272         for (elem = nvlist_next_nvpair(errors, NULL);
4273             elem != NULL;
4274             elem = nvlist_next_nvpair(errors, elem)) {
4275                 (void) snprintf(errbuf, sizeof (errbuf),
4276                     dgettext(TEXT_DOMAIN,
4277                     "cannot release hold from snapshot '%s'"),