7653 tmpfs: calling unlink() on a directory which isn't empty should fail

@@ -19,14 +19,14 @@
  * CDDL HEADER END
  */
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2016 RackTop Systems.
  */
 
-#pragma ident   "%Z%%M% %I%     %E% SMI"
-
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/sysmacros.h>
 #include <sys/systm.h>
 #include <sys/time.h>

@@ -485,10 +485,17 @@
          * is gone, so return that status (same as UFS).
          */
         if (tp != tnp)
                 return (ENOENT);
 
+        /*
+         * If we are unlinking a directory which isn't empty (it has
+         * more than the minumum '.' and '..' entries) return EEXIST.
+         */
+        if (tp->tn_type == VDIR && tp->tn_dirents > 2)
+                return (EEXIST);
+
         tmpfs_hash_out(tpdp);
 
         /*
          * Take tpdp out of the directory list.
          */

@@ -525,11 +532,11 @@
         dir->tn_ctime = now;
         tp->tn_ctime = now;
 
         ASSERT(tp->tn_nlink > 0);
         DECR_COUNT(&tp->tn_nlink, &tp->tn_tlock);
-        if (op == DR_RMDIR && tp->tn_type == VDIR) {
+        if (tp->tn_type == VDIR) {
                 tdirtrunc(tp);
                 ASSERT(tp->tn_nlink == 0);
         }
         return (0);
 }