98 offsetof(dbuf_dirty_record_t, dr_dirty_node));
99 }
100
101 dn->dn_allocated_txg = 0;
102 dn->dn_free_txg = 0;
103 dn->dn_assigned_txg = 0;
104 dn->dn_dirtyctx = 0;
105 dn->dn_dirtyctx_firstset = NULL;
106 dn->dn_bonus = NULL;
107 dn->dn_have_spill = B_FALSE;
108 dn->dn_zio = NULL;
109 dn->dn_oldused = 0;
110 dn->dn_oldflags = 0;
111 dn->dn_olduid = 0;
112 dn->dn_oldgid = 0;
113 dn->dn_newuid = 0;
114 dn->dn_newgid = 0;
115 dn->dn_id_flags = 0;
116
117 dn->dn_dbufs_count = 0;
118 list_create(&dn->dn_dbufs, sizeof (dmu_buf_impl_t),
119 offsetof(dmu_buf_impl_t, db_link));
120
121 dn->dn_moved = 0;
122 return (0);
123 }
124
125 /* ARGSUSED */
126 static void
127 dnode_dest(void *arg, void *unused)
128 {
129 int i;
130 dnode_t *dn = arg;
131
132 rw_destroy(&dn->dn_struct_rwlock);
133 mutex_destroy(&dn->dn_mtx);
134 mutex_destroy(&dn->dn_dbufs_mtx);
135 cv_destroy(&dn->dn_notxholds);
136 refcount_destroy(&dn->dn_holds);
137 refcount_destroy(&dn->dn_tx_holds);
150 ASSERT0(dn->dn_next_blksz[i]);
151 }
152
153 ASSERT0(dn->dn_allocated_txg);
154 ASSERT0(dn->dn_free_txg);
155 ASSERT0(dn->dn_assigned_txg);
156 ASSERT0(dn->dn_dirtyctx);
157 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
158 ASSERT3P(dn->dn_bonus, ==, NULL);
159 ASSERT(!dn->dn_have_spill);
160 ASSERT3P(dn->dn_zio, ==, NULL);
161 ASSERT0(dn->dn_oldused);
162 ASSERT0(dn->dn_oldflags);
163 ASSERT0(dn->dn_olduid);
164 ASSERT0(dn->dn_oldgid);
165 ASSERT0(dn->dn_newuid);
166 ASSERT0(dn->dn_newgid);
167 ASSERT0(dn->dn_id_flags);
168
169 ASSERT0(dn->dn_dbufs_count);
170 list_destroy(&dn->dn_dbufs);
171 }
172
173 void
174 dnode_init(void)
175 {
176 ASSERT(dnode_cache == NULL);
177 dnode_cache = kmem_cache_create("dnode_t",
178 sizeof (dnode_t),
179 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
180 kmem_cache_set_move(dnode_cache, dnode_move);
181 }
182
183 void
184 dnode_fini(void)
185 {
186 kmem_cache_destroy(dnode_cache);
187 dnode_cache = NULL;
188 }
189
455 dn->dn_dirtyctx = 0;
456 if (dn->dn_dirtyctx_firstset != NULL) {
457 kmem_free(dn->dn_dirtyctx_firstset, 1);
458 dn->dn_dirtyctx_firstset = NULL;
459 }
460 if (dn->dn_bonus != NULL) {
461 mutex_enter(&dn->dn_bonus->db_mtx);
462 dbuf_evict(dn->dn_bonus);
463 dn->dn_bonus = NULL;
464 }
465 dn->dn_zio = NULL;
466
467 dn->dn_have_spill = B_FALSE;
468 dn->dn_oldused = 0;
469 dn->dn_oldflags = 0;
470 dn->dn_olduid = 0;
471 dn->dn_oldgid = 0;
472 dn->dn_newuid = 0;
473 dn->dn_newgid = 0;
474 dn->dn_id_flags = 0;
475
476 dmu_zfetch_rele(&dn->dn_zfetch);
477 kmem_cache_free(dnode_cache, dn);
478 arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
479 }
480
481 void
482 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
483 dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
484 {
485 int i;
486
487 if (blocksize == 0)
488 blocksize = 1 << zfs_default_bs;
489 else if (blocksize > SPA_MAXBLOCKSIZE)
490 blocksize = SPA_MAXBLOCKSIZE;
491 else
492 blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
493
494 if (ibs == 0)
685 sizeof (odn->dn_rm_spillblk));
686 bcopy(&odn->dn_next_bonuslen[0], &ndn->dn_next_bonuslen[0],
687 sizeof (odn->dn_next_bonuslen));
688 bcopy(&odn->dn_next_blksz[0], &ndn->dn_next_blksz[0],
689 sizeof (odn->dn_next_blksz));
690 for (i = 0; i < TXG_SIZE; i++) {
691 list_move_tail(&ndn->dn_dirty_records[i],
692 &odn->dn_dirty_records[i]);
693 }
694 bcopy(&odn->dn_ranges[0], &ndn->dn_ranges[0], sizeof (odn->dn_ranges));
695 ndn->dn_allocated_txg = odn->dn_allocated_txg;
696 ndn->dn_free_txg = odn->dn_free_txg;
697 ndn->dn_assigned_txg = odn->dn_assigned_txg;
698 ndn->dn_dirtyctx = odn->dn_dirtyctx;
699 ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
700 ASSERT(refcount_count(&odn->dn_tx_holds) == 0);
701 refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
702 ASSERT(list_is_empty(&ndn->dn_dbufs));
703 list_move_tail(&ndn->dn_dbufs, &odn->dn_dbufs);
704 ndn->dn_dbufs_count = odn->dn_dbufs_count;
705 ndn->dn_bonus = odn->dn_bonus;
706 ndn->dn_have_spill = odn->dn_have_spill;
707 ndn->dn_zio = odn->dn_zio;
708 ndn->dn_oldused = odn->dn_oldused;
709 ndn->dn_oldflags = odn->dn_oldflags;
710 ndn->dn_olduid = odn->dn_olduid;
711 ndn->dn_oldgid = odn->dn_oldgid;
712 ndn->dn_newuid = odn->dn_newuid;
713 ndn->dn_newgid = odn->dn_newgid;
714 ndn->dn_id_flags = odn->dn_id_flags;
715 dmu_zfetch_init(&ndn->dn_zfetch, NULL);
716 list_move_tail(&ndn->dn_zfetch.zf_stream, &odn->dn_zfetch.zf_stream);
717 ndn->dn_zfetch.zf_dnode = odn->dn_zfetch.zf_dnode;
718 ndn->dn_zfetch.zf_stream_cnt = odn->dn_zfetch.zf_stream_cnt;
719 ndn->dn_zfetch.zf_alloc_fail = odn->dn_zfetch.zf_alloc_fail;
720
721 /*
722 * Update back pointers. Updating the handle fixes the back pointer of
723 * every descendant dbuf as well as the bonus dbuf.
724 */
725 ASSERT(ndn->dn_handle->dnh_dnode == odn);
726 ndn->dn_handle->dnh_dnode = ndn;
727 if (ndn->dn_zfetch.zf_dnode == odn) {
728 ndn->dn_zfetch.zf_dnode = ndn;
729 }
730
731 /*
732 * Invalidate the original dnode by clearing all of its back pointers.
733 */
734 odn->dn_dbuf = NULL;
735 odn->dn_handle = NULL;
736 list_create(&odn->dn_dbufs, sizeof (dmu_buf_impl_t),
737 offsetof(dmu_buf_impl_t, db_link));
738 odn->dn_dbufs_count = 0;
739 odn->dn_bonus = NULL;
740 odn->dn_zfetch.zf_dnode = NULL;
741
742 /*
743 * Set the low bit of the objset pointer to ensure that dnode_move()
744 * recognizes the dnode as invalid in any subsequent callback.
745 */
746 POINTER_INVALIDATE(&odn->dn_objset);
747
748 /*
749 * Satisfy the destructor.
750 */
751 for (i = 0; i < TXG_SIZE; i++) {
752 list_create(&odn->dn_dirty_records[i],
753 sizeof (dbuf_dirty_record_t),
754 offsetof(dbuf_dirty_record_t, dr_dirty_node));
755 odn->dn_ranges[i].avl_root = NULL;
756 odn->dn_ranges[i].avl_numnodes = 0;
757 odn->dn_next_nlevels[i] = 0;
758 odn->dn_next_indblkshift[i] = 0;
1506 }
1507 /* there may be no overlap */
1508 rp = nrp;
1509 }
1510 }
1511
1512 void
1513 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
1514 {
1515 dmu_buf_impl_t *db;
1516 uint64_t blkoff, blkid, nblks;
1517 int blksz, blkshift, head, tail;
1518 int trunc = FALSE;
1519 int epbs;
1520
1521 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1522 blksz = dn->dn_datablksz;
1523 blkshift = dn->dn_datablkshift;
1524 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1525
1526 if (len == -1ULL) {
1527 len = UINT64_MAX - off;
1528 trunc = TRUE;
1529 }
1530
1531 /*
1532 * First, block align the region to free:
1533 */
1534 if (ISP2(blksz)) {
1535 head = P2NPHASE(off, blksz);
1536 blkoff = P2PHASE(off, blksz);
1537 if ((off >> blkshift) > dn->dn_maxblkid)
1538 goto out;
1539 } else {
1540 ASSERT(dn->dn_maxblkid == 0);
1541 if (off == 0 && len >= blksz) {
1542 /* Freeing the whole block; fast-track this request */
1543 blkid = 0;
1544 nblks = 1;
1545 goto done;
1546 } else if (off >= blksz) {
|
98 offsetof(dbuf_dirty_record_t, dr_dirty_node));
99 }
100
101 dn->dn_allocated_txg = 0;
102 dn->dn_free_txg = 0;
103 dn->dn_assigned_txg = 0;
104 dn->dn_dirtyctx = 0;
105 dn->dn_dirtyctx_firstset = NULL;
106 dn->dn_bonus = NULL;
107 dn->dn_have_spill = B_FALSE;
108 dn->dn_zio = NULL;
109 dn->dn_oldused = 0;
110 dn->dn_oldflags = 0;
111 dn->dn_olduid = 0;
112 dn->dn_oldgid = 0;
113 dn->dn_newuid = 0;
114 dn->dn_newgid = 0;
115 dn->dn_id_flags = 0;
116
117 dn->dn_dbufs_count = 0;
118 dn->dn_unlisted_l0_blkid = 0;
119 list_create(&dn->dn_dbufs, sizeof (dmu_buf_impl_t),
120 offsetof(dmu_buf_impl_t, db_link));
121
122 dn->dn_moved = 0;
123 return (0);
124 }
125
126 /* ARGSUSED */
127 static void
128 dnode_dest(void *arg, void *unused)
129 {
130 int i;
131 dnode_t *dn = arg;
132
133 rw_destroy(&dn->dn_struct_rwlock);
134 mutex_destroy(&dn->dn_mtx);
135 mutex_destroy(&dn->dn_dbufs_mtx);
136 cv_destroy(&dn->dn_notxholds);
137 refcount_destroy(&dn->dn_holds);
138 refcount_destroy(&dn->dn_tx_holds);
151 ASSERT0(dn->dn_next_blksz[i]);
152 }
153
154 ASSERT0(dn->dn_allocated_txg);
155 ASSERT0(dn->dn_free_txg);
156 ASSERT0(dn->dn_assigned_txg);
157 ASSERT0(dn->dn_dirtyctx);
158 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
159 ASSERT3P(dn->dn_bonus, ==, NULL);
160 ASSERT(!dn->dn_have_spill);
161 ASSERT3P(dn->dn_zio, ==, NULL);
162 ASSERT0(dn->dn_oldused);
163 ASSERT0(dn->dn_oldflags);
164 ASSERT0(dn->dn_olduid);
165 ASSERT0(dn->dn_oldgid);
166 ASSERT0(dn->dn_newuid);
167 ASSERT0(dn->dn_newgid);
168 ASSERT0(dn->dn_id_flags);
169
170 ASSERT0(dn->dn_dbufs_count);
171 ASSERT0(dn->dn_unlisted_l0_blkid);
172 list_destroy(&dn->dn_dbufs);
173 }
174
175 void
176 dnode_init(void)
177 {
178 ASSERT(dnode_cache == NULL);
179 dnode_cache = kmem_cache_create("dnode_t",
180 sizeof (dnode_t),
181 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0);
182 kmem_cache_set_move(dnode_cache, dnode_move);
183 }
184
185 void
186 dnode_fini(void)
187 {
188 kmem_cache_destroy(dnode_cache);
189 dnode_cache = NULL;
190 }
191
457 dn->dn_dirtyctx = 0;
458 if (dn->dn_dirtyctx_firstset != NULL) {
459 kmem_free(dn->dn_dirtyctx_firstset, 1);
460 dn->dn_dirtyctx_firstset = NULL;
461 }
462 if (dn->dn_bonus != NULL) {
463 mutex_enter(&dn->dn_bonus->db_mtx);
464 dbuf_evict(dn->dn_bonus);
465 dn->dn_bonus = NULL;
466 }
467 dn->dn_zio = NULL;
468
469 dn->dn_have_spill = B_FALSE;
470 dn->dn_oldused = 0;
471 dn->dn_oldflags = 0;
472 dn->dn_olduid = 0;
473 dn->dn_oldgid = 0;
474 dn->dn_newuid = 0;
475 dn->dn_newgid = 0;
476 dn->dn_id_flags = 0;
477 dn->dn_unlisted_l0_blkid = 0;
478
479 dmu_zfetch_rele(&dn->dn_zfetch);
480 kmem_cache_free(dnode_cache, dn);
481 arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
482 }
483
484 void
485 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
486 dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx)
487 {
488 int i;
489
490 if (blocksize == 0)
491 blocksize = 1 << zfs_default_bs;
492 else if (blocksize > SPA_MAXBLOCKSIZE)
493 blocksize = SPA_MAXBLOCKSIZE;
494 else
495 blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
496
497 if (ibs == 0)
688 sizeof (odn->dn_rm_spillblk));
689 bcopy(&odn->dn_next_bonuslen[0], &ndn->dn_next_bonuslen[0],
690 sizeof (odn->dn_next_bonuslen));
691 bcopy(&odn->dn_next_blksz[0], &ndn->dn_next_blksz[0],
692 sizeof (odn->dn_next_blksz));
693 for (i = 0; i < TXG_SIZE; i++) {
694 list_move_tail(&ndn->dn_dirty_records[i],
695 &odn->dn_dirty_records[i]);
696 }
697 bcopy(&odn->dn_ranges[0], &ndn->dn_ranges[0], sizeof (odn->dn_ranges));
698 ndn->dn_allocated_txg = odn->dn_allocated_txg;
699 ndn->dn_free_txg = odn->dn_free_txg;
700 ndn->dn_assigned_txg = odn->dn_assigned_txg;
701 ndn->dn_dirtyctx = odn->dn_dirtyctx;
702 ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
703 ASSERT(refcount_count(&odn->dn_tx_holds) == 0);
704 refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
705 ASSERT(list_is_empty(&ndn->dn_dbufs));
706 list_move_tail(&ndn->dn_dbufs, &odn->dn_dbufs);
707 ndn->dn_dbufs_count = odn->dn_dbufs_count;
708 ndn->dn_unlisted_l0_blkid = odn->dn_unlisted_l0_blkid;
709 ndn->dn_bonus = odn->dn_bonus;
710 ndn->dn_have_spill = odn->dn_have_spill;
711 ndn->dn_zio = odn->dn_zio;
712 ndn->dn_oldused = odn->dn_oldused;
713 ndn->dn_oldflags = odn->dn_oldflags;
714 ndn->dn_olduid = odn->dn_olduid;
715 ndn->dn_oldgid = odn->dn_oldgid;
716 ndn->dn_newuid = odn->dn_newuid;
717 ndn->dn_newgid = odn->dn_newgid;
718 ndn->dn_id_flags = odn->dn_id_flags;
719 dmu_zfetch_init(&ndn->dn_zfetch, NULL);
720 list_move_tail(&ndn->dn_zfetch.zf_stream, &odn->dn_zfetch.zf_stream);
721 ndn->dn_zfetch.zf_dnode = odn->dn_zfetch.zf_dnode;
722 ndn->dn_zfetch.zf_stream_cnt = odn->dn_zfetch.zf_stream_cnt;
723 ndn->dn_zfetch.zf_alloc_fail = odn->dn_zfetch.zf_alloc_fail;
724
725 /*
726 * Update back pointers. Updating the handle fixes the back pointer of
727 * every descendant dbuf as well as the bonus dbuf.
728 */
729 ASSERT(ndn->dn_handle->dnh_dnode == odn);
730 ndn->dn_handle->dnh_dnode = ndn;
731 if (ndn->dn_zfetch.zf_dnode == odn) {
732 ndn->dn_zfetch.zf_dnode = ndn;
733 }
734
735 /*
736 * Invalidate the original dnode by clearing all of its back pointers.
737 */
738 odn->dn_dbuf = NULL;
739 odn->dn_handle = NULL;
740 list_create(&odn->dn_dbufs, sizeof (dmu_buf_impl_t),
741 offsetof(dmu_buf_impl_t, db_link));
742 odn->dn_dbufs_count = 0;
743 odn->dn_unlisted_l0_blkid = 0;
744 odn->dn_bonus = NULL;
745 odn->dn_zfetch.zf_dnode = NULL;
746
747 /*
748 * Set the low bit of the objset pointer to ensure that dnode_move()
749 * recognizes the dnode as invalid in any subsequent callback.
750 */
751 POINTER_INVALIDATE(&odn->dn_objset);
752
753 /*
754 * Satisfy the destructor.
755 */
756 for (i = 0; i < TXG_SIZE; i++) {
757 list_create(&odn->dn_dirty_records[i],
758 sizeof (dbuf_dirty_record_t),
759 offsetof(dbuf_dirty_record_t, dr_dirty_node));
760 odn->dn_ranges[i].avl_root = NULL;
761 odn->dn_ranges[i].avl_numnodes = 0;
762 odn->dn_next_nlevels[i] = 0;
763 odn->dn_next_indblkshift[i] = 0;
1511 }
1512 /* there may be no overlap */
1513 rp = nrp;
1514 }
1515 }
1516
1517 void
1518 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
1519 {
1520 dmu_buf_impl_t *db;
1521 uint64_t blkoff, blkid, nblks;
1522 int blksz, blkshift, head, tail;
1523 int trunc = FALSE;
1524 int epbs;
1525
1526 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1527 blksz = dn->dn_datablksz;
1528 blkshift = dn->dn_datablkshift;
1529 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1530
1531 if (len == DMU_OBJECT_END) {
1532 len = UINT64_MAX - off;
1533 trunc = TRUE;
1534 }
1535
1536 /*
1537 * First, block align the region to free:
1538 */
1539 if (ISP2(blksz)) {
1540 head = P2NPHASE(off, blksz);
1541 blkoff = P2PHASE(off, blksz);
1542 if ((off >> blkshift) > dn->dn_maxblkid)
1543 goto out;
1544 } else {
1545 ASSERT(dn->dn_maxblkid == 0);
1546 if (off == 0 && len >= blksz) {
1547 /* Freeing the whole block; fast-track this request */
1548 blkid = 0;
1549 nblks = 1;
1550 goto done;
1551 } else if (off >= blksz) {
|