7656 unlinking directory on tmpfs can cause kernel panic

@@ -19,14 +19,14 @@
  * CDDL HEADER END
  */
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2016 RackTop Systems.
  */
 
-#pragma ident   "%Z%%M% %I%     %E% SMI"
-
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/sysmacros.h>
 #include <sys/systm.h>
 #include <sys/time.h>

@@ -523,16 +523,24 @@
         gethrestime(&now);
         dir->tn_mtime = now;
         dir->tn_ctime = now;
         tp->tn_ctime = now;
 
+        /*
+         * If this is a _REMOVE (unlink) operation there may
+         * be other links to the directory entry.
+         */
         ASSERT(tp->tn_nlink > 0);
         DECR_COUNT(&tp->tn_nlink, &tp->tn_tlock);
-        if (op == DR_RMDIR && tp->tn_type == VDIR) {
+        if (op == DR_RMDIR || (op == DR_REMOVE && tp->tn_type == VDIR)) {
+                if (tp->tn_nlink > 1) {
+                        ASSERT(op == DR_REMOVE);
+                } else {
                 tdirtrunc(tp);
                 ASSERT(tp->tn_nlink == 0);
         }
+        }
         return (0);
 }
 
 /*
  * tdirinit is used internally to initialize a directory (dir)

@@ -613,10 +621,11 @@
         struct tdirent *tdp;
         struct tmpnode *tp;
         size_t namelen;
         timestruc_t now;
         int isvattrdir, isdotdot, skip_decr;
+        int lock_held;
 
         ASSERT(RW_WRITE_HELD(&dir->tn_rwlock));
         ASSERT(dir->tn_type == VDIR);
 
         isvattrdir = (dir->tn_vnode->v_flag & V_XATTRDIR) ? 1 : 0;

@@ -651,11 +660,34 @@
                 tmpfs_hash_out(tdp);
 
                 tmp_memfree(tdp, sizeof (struct tdirent) + namelen);
                 dir->tn_size -= (sizeof (struct tdirent) + namelen);
                 dir->tn_dirents--;
+
+                /*
+                 * This directory entry may itself be a directory with
+                 * entries and removing it may have created orphans.
+                 * On a normal filesystem like UFS this wouldn't be
+                 * a huge problem because fcsk can reclaim them.  For
+                 * TMPFS which resides in RAM however, it means we
+                 * end up leaking memory.
+                 *
+                 * To avoid this we also truncate child directories,
+                 * but only if they have no other links to them.
+                 */
+                if (!isdotdot && tp->tn_type == VDIR && tp != dir) {
+                        if (tp->tn_nlink > 1)
+                                continue;
+                        lock_held = RW_WRITE_HELD(&tp->tn_rwlock);
+                        if (!lock_held)
+                                rw_enter(&tp->tn_rwlock, RW_WRITER);
+                        tdirtrunc(tp);
+                        if (!lock_held)
+                                rw_exit(&tp->tn_rwlock);
+                        ASSERT(tp->tn_nlink == 0);
         }
+        }
 
         gethrestime(&now);
         dir->tn_mtime = now;
         dir->tn_ctime = now;