7656 unlinking directory on tmpfs can cause kernel panic

*** 19,32 ****
   * CDDL HEADER END
   */
  /*
   * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
   */
  
- #pragma ident   "%Z%%M% %I%     %E% SMI"
- 
  #include <sys/types.h>
  #include <sys/param.h>
  #include <sys/sysmacros.h>
  #include <sys/systm.h>
  #include <sys/time.h>
--- 19,32 ----
   * CDDL HEADER END
   */
  /*
   * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
+  *
+  * Copyright 2016 RackTop Systems.
   */
  
  #include <sys/types.h>
  #include <sys/param.h>
  #include <sys/sysmacros.h>
  #include <sys/systm.h>
  #include <sys/time.h>
*** 523,538 ****
          gethrestime(&now);
          dir->tn_mtime = now;
          dir->tn_ctime = now;
          tp->tn_ctime = now;
  
          ASSERT(tp->tn_nlink > 0);
          DECR_COUNT(&tp->tn_nlink, &tp->tn_tlock);
!         if (op == DR_RMDIR && tp->tn_type == VDIR) {
                  tdirtrunc(tp);
                  ASSERT(tp->tn_nlink == 0);
          }
          return (0);
  }
  
  /*
   * tdirinit is used internally to initialize a directory (dir)
--- 523,546 ----
          gethrestime(&now);
          dir->tn_mtime = now;
          dir->tn_ctime = now;
          tp->tn_ctime = now;
  
+         /*
+          * If this is a _REMOVE (unlink) operation there may
+          * be other links to the directory entry.
+          */
          ASSERT(tp->tn_nlink > 0);
          DECR_COUNT(&tp->tn_nlink, &tp->tn_tlock);
!         if (op == DR_RMDIR || (op == DR_REMOVE && tp->tn_type == VDIR)) {
!                 if (tp->tn_nlink > 1) {
!                         ASSERT(op == DR_REMOVE);
!                 } else {
                          tdirtrunc(tp);
                          ASSERT(tp->tn_nlink == 0);
                  }
+         }
          return (0);
  }
  
  /*
   * tdirinit is used internally to initialize a directory (dir)
*** 613,622 ****
--- 621,631 ----
          struct tdirent *tdp;
          struct tmpnode *tp;
          size_t namelen;
          timestruc_t now;
          int isvattrdir, isdotdot, skip_decr;
+         int lock_held;
  
          ASSERT(RW_WRITE_HELD(&dir->tn_rwlock));
          ASSERT(dir->tn_type == VDIR);
  
          isvattrdir = (dir->tn_vnode->v_flag & V_XATTRDIR) ? 1 : 0;
*** 651,661 ****
--- 660,693 ----
                  tmpfs_hash_out(tdp);
  
                  tmp_memfree(tdp, sizeof (struct tdirent) + namelen);
                  dir->tn_size -= (sizeof (struct tdirent) + namelen);
                  dir->tn_dirents--;
+ 
+                 /*
+                  * This directory entry may itself be a directory with
+                  * entries and removing it may have created orphans.
+                  * On a normal filesystem like UFS this wouldn't be
+                  * a huge problem because fcsk can reclaim them.  For
+                  * TMPFS which resides in RAM however, it means we
+                  * end up leaking memory.
+                  *
+                  * To avoid this we also truncate child directories,
+                  * but only if they have no other links to them.
+                  */
+                 if (!isdotdot && tp->tn_type == VDIR && tp != dir) {
+                         if (tp->tn_nlink > 1)
+                                 continue;
+                         lock_held = RW_WRITE_HELD(&tp->tn_rwlock);
+                         if (!lock_held)
+                                 rw_enter(&tp->tn_rwlock, RW_WRITER);
+                         tdirtrunc(tp);
+                         if (!lock_held)
+                                 rw_exit(&tp->tn_rwlock);
+                         ASSERT(tp->tn_nlink == 0);
                  }
+         }
  
          gethrestime(&now);
          dir->tn_mtime = now;
          dir->tn_ctime = now;