6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
24 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
25 * Copyright (c) 2014 Integros [integros.com]
26 */
27
28 #include <sys/zfs_context.h>
29 #include <sys/dbuf.h>
30 #include <sys/dnode.h>
31 #include <sys/dmu.h>
32 #include <sys/dmu_impl.h>
33 #include <sys/dmu_tx.h>
34 #include <sys/dmu_objset.h>
35 #include <sys/dsl_dir.h>
36 #include <sys/dsl_dataset.h>
37 #include <sys/spa.h>
38 #include <sys/zio.h>
39 #include <sys/dmu_zfetch.h>
40 #include <sys/range_tree.h>
41
42 static kmem_cache_t *dnode_cache;
43 /*
44 * Define DNODE_STATS to turn on statistic gathering. By default, it is only
45 * turned on when DEBUG is also defined.
46 */
47 #ifdef DEBUG
48 #define DNODE_STATS
49 #endif /* DEBUG */
50
51 #ifdef DNODE_STATS
52 #define DNODE_STAT_ADD(stat) ((stat)++)
53 #else
54 #define DNODE_STAT_ADD(stat) /* nothing */
55 #endif /* DNODE_STATS */
56
57 static dnode_phys_t dnode_phys_zero;
58
59 int zfs_default_bs = SPA_MINBLOCKSHIFT;
60 int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
61
62 static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
63
64 static int
65 dbuf_compare(const void *x1, const void *x2)
66 {
67 const dmu_buf_impl_t *d1 = x1;
68 const dmu_buf_impl_t *d2 = x2;
69
70 if (d1->db_level < d2->db_level) {
71 return (-1);
72 }
73 if (d1->db_level > d2->db_level) {
74 return (1);
75 }
76
77 if (d1->db_blkid < d2->db_blkid) {
78 return (-1);
79 }
80 if (d1->db_blkid > d2->db_blkid) {
81 return (1);
82 }
196 ASSERT3P(dn->dn_zio, ==, NULL);
197 ASSERT0(dn->dn_oldused);
198 ASSERT0(dn->dn_oldflags);
199 ASSERT0(dn->dn_olduid);
200 ASSERT0(dn->dn_oldgid);
201 ASSERT0(dn->dn_newuid);
202 ASSERT0(dn->dn_newgid);
203 ASSERT0(dn->dn_id_flags);
204
205 ASSERT0(dn->dn_dbufs_count);
206 avl_destroy(&dn->dn_dbufs);
207 }
208
209 void
210 dnode_init(void)
211 {
212 ASSERT(dnode_cache == NULL);
213 dnode_cache = kmem_cache_create("dnode_t",
214 sizeof (dnode_t),
215 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
216 kmem_cache_set_move(dnode_cache, dnode_move);
217 }
218
219 void
220 dnode_fini(void)
221 {
222 kmem_cache_destroy(dnode_cache);
223 dnode_cache = NULL;
224 }
225
226
227 #ifdef ZFS_DEBUG
228 void
229 dnode_verify(dnode_t *dn)
230 {
231 int drop_struct_lock = FALSE;
232
233 ASSERT(dn->dn_phys);
234 ASSERT(dn->dn_objset);
235 ASSERT(dn->dn_handle->dnh_dnode == dn);
236
680 DN_MAX_BONUSLEN - (dn->dn_nblkptr-1) * sizeof (blkptr_t);
681 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
682 }
683
684 dn->dn_allocated_txg = tx->tx_txg;
685 mutex_exit(&dn->dn_mtx);
686 }
687
688 #ifdef DNODE_STATS
689 static struct {
690 uint64_t dms_dnode_invalid;
691 uint64_t dms_dnode_recheck1;
692 uint64_t dms_dnode_recheck2;
693 uint64_t dms_dnode_special;
694 uint64_t dms_dnode_handle;
695 uint64_t dms_dnode_rwlock;
696 uint64_t dms_dnode_active;
697 } dnode_move_stats;
698 #endif /* DNODE_STATS */
699
700 static void
701 dnode_move_impl(dnode_t *odn, dnode_t *ndn)
702 {
703 int i;
704
705 ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock));
706 ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx));
707 ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx));
708 ASSERT(!RW_LOCK_HELD(&odn->dn_zfetch.zf_rwlock));
709
710 /* Copy fields. */
711 ndn->dn_objset = odn->dn_objset;
712 ndn->dn_object = odn->dn_object;
713 ndn->dn_dbuf = odn->dn_dbuf;
714 ndn->dn_handle = odn->dn_handle;
715 ndn->dn_phys = odn->dn_phys;
716 ndn->dn_type = odn->dn_type;
717 ndn->dn_bonuslen = odn->dn_bonuslen;
718 ndn->dn_bonustype = odn->dn_bonustype;
719 ndn->dn_nblkptr = odn->dn_nblkptr;
816 odn->dn_assigned_txg = 0;
817 odn->dn_dirtyctx = 0;
818 odn->dn_dirtyctx_firstset = NULL;
819 odn->dn_have_spill = B_FALSE;
820 odn->dn_zio = NULL;
821 odn->dn_oldused = 0;
822 odn->dn_oldflags = 0;
823 odn->dn_olduid = 0;
824 odn->dn_oldgid = 0;
825 odn->dn_newuid = 0;
826 odn->dn_newgid = 0;
827 odn->dn_id_flags = 0;
828
829 /*
830 * Mark the dnode.
831 */
832 ndn->dn_moved = 1;
833 odn->dn_moved = (uint8_t)-1;
834 }
835
836 #ifdef _KERNEL
837 /*ARGSUSED*/
838 static kmem_cbrc_t
839 dnode_move(void *buf, void *newbuf, size_t size, void *arg)
840 {
841 dnode_t *odn = buf, *ndn = newbuf;
842 objset_t *os;
843 int64_t refcount;
844 uint32_t dbufs;
845
846 /*
847 * The dnode is on the objset's list of known dnodes if the objset
848 * pointer is valid. We set the low bit of the objset pointer when
849 * freeing the dnode to invalidate it, and the memory patterns written
850 * by kmem (baddcafe and deadbeef) set at least one of the two low bits.
851 * A newly created dnode sets the objset pointer last of all to indicate
852 * that the dnode is known and in a valid state to be moved by this
853 * function.
854 */
855 os = odn->dn_objset;
856 if (!POINTER_IS_VALID(os)) {
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
24 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
25 * Copyright (c) 2014 Integros [integros.com]
26 * Copyright 2017 RackTop Systems.
27 */
28
29 #include <sys/zfs_context.h>
30 #include <sys/dbuf.h>
31 #include <sys/dnode.h>
32 #include <sys/dmu.h>
33 #include <sys/dmu_impl.h>
34 #include <sys/dmu_tx.h>
35 #include <sys/dmu_objset.h>
36 #include <sys/dsl_dir.h>
37 #include <sys/dsl_dataset.h>
38 #include <sys/spa.h>
39 #include <sys/zio.h>
40 #include <sys/dmu_zfetch.h>
41 #include <sys/range_tree.h>
42
43 static kmem_cache_t *dnode_cache;
44 /*
45 * Define DNODE_STATS to turn on statistic gathering. By default, it is only
46 * turned on when DEBUG is also defined.
47 */
48 #ifdef DEBUG
49 #define DNODE_STATS
50 #endif /* DEBUG */
51
52 #ifdef DNODE_STATS
53 #define DNODE_STAT_ADD(stat) ((stat)++)
54 #else
55 #define DNODE_STAT_ADD(stat) /* nothing */
56 #endif /* DNODE_STATS */
57
58 static dnode_phys_t dnode_phys_zero;
59
60 int zfs_default_bs = SPA_MINBLOCKSHIFT;
61 int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
62
63 #ifdef _KERNEL
64 static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
65 #endif /* _KERNEL */
66
67 static int
68 dbuf_compare(const void *x1, const void *x2)
69 {
70 const dmu_buf_impl_t *d1 = x1;
71 const dmu_buf_impl_t *d2 = x2;
72
73 if (d1->db_level < d2->db_level) {
74 return (-1);
75 }
76 if (d1->db_level > d2->db_level) {
77 return (1);
78 }
79
80 if (d1->db_blkid < d2->db_blkid) {
81 return (-1);
82 }
83 if (d1->db_blkid > d2->db_blkid) {
84 return (1);
85 }
199 ASSERT3P(dn->dn_zio, ==, NULL);
200 ASSERT0(dn->dn_oldused);
201 ASSERT0(dn->dn_oldflags);
202 ASSERT0(dn->dn_olduid);
203 ASSERT0(dn->dn_oldgid);
204 ASSERT0(dn->dn_newuid);
205 ASSERT0(dn->dn_newgid);
206 ASSERT0(dn->dn_id_flags);
207
208 ASSERT0(dn->dn_dbufs_count);
209 avl_destroy(&dn->dn_dbufs);
210 }
211
212 void
213 dnode_init(void)
214 {
215 ASSERT(dnode_cache == NULL);
216 dnode_cache = kmem_cache_create("dnode_t",
217 sizeof (dnode_t),
218 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
219 #ifdef _KERNEL
220 kmem_cache_set_move(dnode_cache, dnode_move);
221 #endif /* _KERNEL */
222 }
223
224 void
225 dnode_fini(void)
226 {
227 kmem_cache_destroy(dnode_cache);
228 dnode_cache = NULL;
229 }
230
231
232 #ifdef ZFS_DEBUG
233 void
234 dnode_verify(dnode_t *dn)
235 {
236 int drop_struct_lock = FALSE;
237
238 ASSERT(dn->dn_phys);
239 ASSERT(dn->dn_objset);
240 ASSERT(dn->dn_handle->dnh_dnode == dn);
241
685 DN_MAX_BONUSLEN - (dn->dn_nblkptr-1) * sizeof (blkptr_t);
686 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
687 }
688
689 dn->dn_allocated_txg = tx->tx_txg;
690 mutex_exit(&dn->dn_mtx);
691 }
692
693 #ifdef DNODE_STATS
694 static struct {
695 uint64_t dms_dnode_invalid;
696 uint64_t dms_dnode_recheck1;
697 uint64_t dms_dnode_recheck2;
698 uint64_t dms_dnode_special;
699 uint64_t dms_dnode_handle;
700 uint64_t dms_dnode_rwlock;
701 uint64_t dms_dnode_active;
702 } dnode_move_stats;
703 #endif /* DNODE_STATS */
704
705 #ifdef _KERNEL
706 static void
707 dnode_move_impl(dnode_t *odn, dnode_t *ndn)
708 {
709 int i;
710
711 ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock));
712 ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx));
713 ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx));
714 ASSERT(!RW_LOCK_HELD(&odn->dn_zfetch.zf_rwlock));
715
716 /* Copy fields. */
717 ndn->dn_objset = odn->dn_objset;
718 ndn->dn_object = odn->dn_object;
719 ndn->dn_dbuf = odn->dn_dbuf;
720 ndn->dn_handle = odn->dn_handle;
721 ndn->dn_phys = odn->dn_phys;
722 ndn->dn_type = odn->dn_type;
723 ndn->dn_bonuslen = odn->dn_bonuslen;
724 ndn->dn_bonustype = odn->dn_bonustype;
725 ndn->dn_nblkptr = odn->dn_nblkptr;
822 odn->dn_assigned_txg = 0;
823 odn->dn_dirtyctx = 0;
824 odn->dn_dirtyctx_firstset = NULL;
825 odn->dn_have_spill = B_FALSE;
826 odn->dn_zio = NULL;
827 odn->dn_oldused = 0;
828 odn->dn_oldflags = 0;
829 odn->dn_olduid = 0;
830 odn->dn_oldgid = 0;
831 odn->dn_newuid = 0;
832 odn->dn_newgid = 0;
833 odn->dn_id_flags = 0;
834
835 /*
836 * Mark the dnode.
837 */
838 ndn->dn_moved = 1;
839 odn->dn_moved = (uint8_t)-1;
840 }
841
842 /*ARGSUSED*/
843 static kmem_cbrc_t
844 dnode_move(void *buf, void *newbuf, size_t size, void *arg)
845 {
846 dnode_t *odn = buf, *ndn = newbuf;
847 objset_t *os;
848 int64_t refcount;
849 uint32_t dbufs;
850
851 /*
852 * The dnode is on the objset's list of known dnodes if the objset
853 * pointer is valid. We set the low bit of the objset pointer when
854 * freeing the dnode to invalidate it, and the memory patterns written
855 * by kmem (baddcafe and deadbeef) set at least one of the two low bits.
856 * A newly created dnode sets the objset pointer last of all to indicate
857 * that the dnode is known and in a valid state to be moved by this
858 * function.
859 */
860 os = odn->dn_objset;
861 if (!POINTER_IS_VALID(os)) {
|