arc_get_data_buf should be more aggressive in eviction when memory is unavailable

*** 2517,2542 ****
          uint64_t                size = buf->b_hdr->b_size;
          arc_buf_contents_t      type = buf->b_hdr->b_type;
  
          arc_adapt(size, state);
  
          /*
           * We have not yet reached cache maximum size,
           * just allocate a new buffer.
           */
          if (!arc_evict_needed(type)) {
                  if (type == ARC_BUFC_METADATA) {
!                         buf->b_data = zio_buf_alloc(size);
                          arc_space_consume(size, ARC_SPACE_DATA);
                  } else {
                          ASSERT(type == ARC_BUFC_DATA);
!                         buf->b_data = zio_data_buf_alloc(size);
                          ARCSTAT_INCR(arcstat_data_size, size);
                          atomic_add_64(&arc_size, size);
-                 }
                  goto out;
          }
  
          /*
           * If we are prefetching from the mfu ghost list, this buffer
           * will end up on the mru list; so steal space from there.
           */
--- 2517,2552 ----
          uint64_t                size = buf->b_hdr->b_size;
          arc_buf_contents_t      type = buf->b_hdr->b_type;
  
          arc_adapt(size, state);
  
+ top:
          /*
           * We have not yet reached cache maximum size,
           * just allocate a new buffer.
           */
          if (!arc_evict_needed(type)) {
                  if (type == ARC_BUFC_METADATA) {
!                         buf->b_data = zio_buf_alloc_canfail(size);
!                         if (buf->b_data != NULL) {
                                  arc_space_consume(size, ARC_SPACE_DATA);
+                                 goto out;
+                         }
                  } else {
                          ASSERT(type == ARC_BUFC_DATA);
!                         buf->b_data = zio_data_buf_alloc_canfail(size);
!                         if (buf->b_data != NULL) {
                                  ARCSTAT_INCR(arcstat_data_size, size);
                                  atomic_add_64(&arc_size, size);
                                  goto out;
                          }
+                 }
+                 /*
+                  * Memory allocation failed probably due to excessive
+                  * fragmentation, we need to evict regardless.
+                  */
+         }
  
          /*
           * If we are prefetching from the mfu ghost list, this buffer
           * will end up on the mru list; so steal space from there.
           */
*** 2554,2573 ****
                  uint64_t mfu_space = arc_c - arc_p;
                  state =  (arc_mru->arcs_lsize[type] >= size &&
                      mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
          }
          if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
-                 if (type == ARC_BUFC_METADATA) {
-                         buf->b_data = zio_buf_alloc(size);
-                         arc_space_consume(size, ARC_SPACE_DATA);
-                 } else {
-                         ASSERT(type == ARC_BUFC_DATA);
-                         buf->b_data = zio_data_buf_alloc(size);
-                         ARCSTAT_INCR(arcstat_data_size, size);
-                         atomic_add_64(&arc_size, size);
-                 }
                  ARCSTAT_BUMP(arcstat_recycle_miss);
          }
          ASSERT(buf->b_data != NULL);
  out:
          /*
           * Update the state size.  Note that ghost states have a
--- 2564,2575 ----
                  uint64_t mfu_space = arc_c - arc_p;
                  state =  (arc_mru->arcs_lsize[type] >= size &&
                      mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
          }
          if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
                  ARCSTAT_BUMP(arcstat_recycle_miss);
+                 goto top;
          }
          ASSERT(buf->b_data != NULL);
  out:
          /*
           * Update the state size.  Note that ghost states have a