3995 Memory leak of compressed buffers in l2arc_write_done
3997 ZFS L2ARC default behavior should allow reading while writing

@@ -292,11 +292,10 @@
         kstat_named_t arcstat_l2_read_bytes;
         kstat_named_t arcstat_l2_write_bytes;
         kstat_named_t arcstat_l2_writes_sent;
         kstat_named_t arcstat_l2_writes_done;
         kstat_named_t arcstat_l2_writes_error;
-        kstat_named_t arcstat_l2_writes_hdr_miss;
         kstat_named_t arcstat_l2_evict_lock_retry;
         kstat_named_t arcstat_l2_evict_reading;
         kstat_named_t arcstat_l2_free_on_write;
         kstat_named_t arcstat_l2_abort_lowmem;
         kstat_named_t arcstat_l2_cksum_bad;

@@ -358,11 +357,10 @@
         { "l2_read_bytes",              KSTAT_DATA_UINT64 },
         { "l2_write_bytes",             KSTAT_DATA_UINT64 },
         { "l2_writes_sent",             KSTAT_DATA_UINT64 },
         { "l2_writes_done",             KSTAT_DATA_UINT64 },
         { "l2_writes_error",            KSTAT_DATA_UINT64 },
-        { "l2_writes_hdr_miss",         KSTAT_DATA_UINT64 },
         { "l2_evict_lock_retry",        KSTAT_DATA_UINT64 },
         { "l2_evict_reading",           KSTAT_DATA_UINT64 },
         { "l2_free_on_write",           KSTAT_DATA_UINT64 },
         { "l2_abort_lowmem",            KSTAT_DATA_UINT64 },
         { "l2_cksum_bad",               KSTAT_DATA_UINT64 },

@@ -620,11 +618,11 @@
 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;     /* interval seconds */
 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
 boolean_t l2arc_noprefetch = B_TRUE;            /* don't cache prefetch bufs */
 boolean_t l2arc_feed_again = B_TRUE;            /* turbo warmup */
-boolean_t l2arc_norw = B_TRUE;                  /* no reads during writes */
+boolean_t l2arc_norw = B_FALSE;                 /* no reads during writes */
 
 /*
  * L2ARC Internals
  */
 typedef struct l2arc_dev {

@@ -4146,11 +4144,11 @@
 {
         l2arc_write_callback_t *cb;
         l2arc_dev_t *dev;
         list_t *buflist;
         arc_buf_hdr_t *head, *ab, *ab_prev;
-        l2arc_buf_hdr_t *abl2;
+        l2arc_buf_hdr_t *l2hdr;
         kmutex_t *hash_lock;
 
         cb = zio->io_private;
         ASSERT(cb != NULL);
         dev = cb->l2wcb_dev;

@@ -4168,40 +4166,33 @@
         mutex_enter(&l2arc_buflist_mtx);
 
         /*
          * All writes completed, or an error was hit.
          */
+        dev->l2ad_writing = B_FALSE;
         for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
                 ab_prev = list_prev(buflist, ab);
 
                 hash_lock = HDR_LOCK(ab);
-                if (!mutex_tryenter(hash_lock)) {
-                        /*
-                         * This buffer misses out.  It may be in a stage
-                         * of eviction.  Its ARC_L2_WRITING flag will be
-                         * left set, denying reads to this buffer.
-                         */
-                        ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
-                        continue;
-                }
+                mutex_enter(hash_lock);
 
-                abl2 = ab->b_l2hdr;
+                l2hdr = ab->b_l2hdr;
 
                 /*
                  * Release the temporary compressed buffer as soon as possible.
                  */
-                if (abl2->b_compress != ZIO_COMPRESS_OFF)
+                if (l2hdr->b_compress != ZIO_COMPRESS_OFF)
                         l2arc_release_cdata_buf(ab);
 
                 if (zio->io_error != 0) {
                         /*
                          * Error - drop L2ARC entry.
                          */
                         list_remove(buflist, ab);
-                        ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+                        ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
                         ab->b_l2hdr = NULL;
-                        kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
+                        kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
                         ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
                 }
 
                 /*
                  * Allow ARC to begin reads to this L2ARC entry.

@@ -4348,11 +4339,11 @@
  */
 static void
 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 {
         list_t *buflist;
-        l2arc_buf_hdr_t *abl2;
+        l2arc_buf_hdr_t *l2hdr;
         arc_buf_hdr_t *ab, *ab_prev;
         kmutex_t *hash_lock;
         uint64_t taddr;
 
         buflist = dev->l2ad_buflist;

@@ -4448,14 +4439,14 @@
 
                         /*
                          * Tell ARC this no longer exists in L2ARC.
                          */
                         if (ab->b_l2hdr != NULL) {
-                                abl2 = ab->b_l2hdr;
-                                ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+                                l2hdr = ab->b_l2hdr;
+                                ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
                                 ab->b_l2hdr = NULL;
-                                kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
+                                kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
                                 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
                         }
                         list_remove(buflist, ab);
 
                         /*

@@ -4723,13 +4714,14 @@
                 dev->l2ad_hand = dev->l2ad_start;
                 dev->l2ad_evict = dev->l2ad_start;
                 dev->l2ad_first = B_FALSE;
         }
 
+        /* dev->l2ad_writing will be lowered in the zio done callback */
         dev->l2ad_writing = B_TRUE;
         (void) zio_wait(pio);
-        dev->l2ad_writing = B_FALSE;
+        ASSERT(dev->l2ad_writing == B_FALSE);
 
         return (write_asize);
 }
 
 /*