1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  24  * Copyright (c) 2013 by Delphix. All rights reserved.
  25  */
  26 
  27 /*
  28  * DVA-based Adjustable Replacement Cache
  29  *
  30  * While much of the theory of operation used here is
  31  * based on the self-tuning, low overhead replacement cache
  32  * presented by Megiddo and Modha at FAST 2003, there are some
  33  * significant differences:
  34  *
  35  * 1. The Megiddo and Modha model assumes any page is evictable.
  36  * Pages in its cache cannot be "locked" into memory.  This makes
  37  * the eviction algorithm simple: evict the last page in the list.
  38  * This also make the performance characteristics easy to reason
  39  * about.  Our cache is not so simple.  At any given moment, some
  40  * subset of the blocks in the cache are un-evictable because we
  41  * have handed out a reference to them.  Blocks are only evictable
  42  * when there are no external references active.  This makes
  43  * eviction far more problematic:  we choose to evict the evictable
  44  * blocks that are the "lowest" in the list.
  45  *
  46  * There are times when it is not possible to evict the requested
  47  * space.  In these circumstances we are unable to adjust the cache
  48  * size.  To prevent the cache growing unbounded at these times we
  49  * implement a "cache throttle" that slows the flow of new data
  50  * into the cache until we can make space available.
  51  *
  52  * 2. The Megiddo and Modha model assumes a fixed cache size.
  53  * Pages are evicted when the cache is full and there is a cache
  54  * miss.  Our model has a variable sized cache.  It grows with
  55  * high use, but also tries to react to memory pressure from the
  56  * operating system: decreasing its size when system memory is
  57  * tight.
  58  *
  59  * 3. The Megiddo and Modha model assumes a fixed page size. All
  60  * elements of the cache are therefor exactly the same size.  So
  61  * when adjusting the cache size following a cache miss, its simply
  62  * a matter of choosing a single page to evict.  In our model, we
  63  * have variable sized cache blocks (rangeing from 512 bytes to
  64  * 128K bytes).  We therefor choose a set of blocks to evict to make
  65  * space for a cache miss that approximates as closely as possible
  66  * the space used by the new block.
  67  *
  68  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
  69  * by N. Megiddo & D. Modha, FAST 2003
  70  */
  71 
  72 /*
  73  * The locking model:
  74  *
  75  * A new reference to a cache buffer can be obtained in two
  76  * ways: 1) via a hash table lookup using the DVA as a key,
  77  * or 2) via one of the ARC lists.  The arc_read() interface
  78  * uses method 1, while the internal arc algorithms for
  79  * adjusting the cache use method 2.  We therefor provide two
  80  * types of locks: 1) the hash table lock array, and 2) the
  81  * arc list locks.
  82  *
  83  * Buffers do not have their own mutexes, rather they rely on the
  84  * hash table mutexes for the bulk of their protection (i.e. most
  85  * fields in the arc_buf_hdr_t are protected by these mutexes).
  86  *
  87  * buf_hash_find() returns the appropriate mutex (held) when it
  88  * locates the requested buffer in the hash table.  It returns
  89  * NULL for the mutex if the buffer was not in the table.
  90  *
  91  * buf_hash_remove() expects the appropriate hash mutex to be
  92  * already held before it is invoked.
  93  *
  94  * Each arc state also has a mutex which is used to protect the
  95  * buffer list associated with the state.  When attempting to
  96  * obtain a hash table lock while holding an arc list lock you
  97  * must use: mutex_tryenter() to avoid deadlock.  Also note that
  98  * the active state mutex must be held before the ghost state mutex.
  99  *
 100  * Arc buffers may have an associated eviction callback function.
 101  * This function will be invoked prior to removing the buffer (e.g.
 102  * in arc_do_user_evicts()).  Note however that the data associated
 103  * with the buffer may be evicted prior to the callback.  The callback
 104  * must be made with *no locks held* (to prevent deadlock).  Additionally,
 105  * the users of callbacks must ensure that their private data is
 106  * protected from simultaneous callbacks from arc_buf_evict()
 107  * and arc_do_user_evicts().
 108  *
 109  * Note that the majority of the performance stats are manipulated
 110  * with atomic operations.
 111  *
 112  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
 113  *
 114  *      - L2ARC buflist creation
 115  *      - L2ARC buflist eviction
 116  *      - L2ARC write completion, which walks L2ARC buflists
 117  *      - ARC header destruction, as it removes from L2ARC buflists
 118  *      - ARC header release, as it removes from L2ARC buflists
 119  */
 120 
 121 #include <sys/spa.h>
 122 #include <sys/zio.h>
 123 #include <sys/zfs_context.h>
 124 #include <sys/arc.h>
 125 #include <sys/refcount.h>
 126 #include <sys/vdev.h>
 127 #include <sys/vdev_impl.h>
 128 #ifdef _KERNEL
 129 #include <sys/vmsystm.h>
 130 #include <vm/anon.h>
 131 #include <sys/fs/swapnode.h>
 132 #include <sys/dnlc.h>
 133 #endif
 134 #include <sys/callb.h>
 135 #include <sys/kstat.h>
 136 #include <zfs_fletcher.h>
 137 
 138 #ifndef _KERNEL
 139 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
 140 boolean_t arc_watch = B_FALSE;
 141 int arc_procfd;
 142 #endif
 143 
 144 static kmutex_t         arc_reclaim_thr_lock;
 145 static kcondvar_t       arc_reclaim_thr_cv;     /* used to signal reclaim thr */
 146 static uint8_t          arc_thread_exit;
 147 
 148 extern int zfs_write_limit_shift;
 149 extern uint64_t zfs_write_limit_max;
 150 extern kmutex_t zfs_write_limit_lock;
 151 
 152 #define ARC_REDUCE_DNLC_PERCENT 3
 153 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
 154 
 155 typedef enum arc_reclaim_strategy {
 156         ARC_RECLAIM_AGGR,               /* Aggressive reclaim strategy */
 157         ARC_RECLAIM_CONS                /* Conservative reclaim strategy */
 158 } arc_reclaim_strategy_t;
 159 
 160 /* number of seconds before growing cache again */
 161 static int              arc_grow_retry = 60;
 162 
 163 /* shift of arc_c for calculating both min and max arc_p */
 164 static int              arc_p_min_shift = 4;
 165 
 166 /* log2(fraction of arc to reclaim) */
 167 static int              arc_shrink_shift = 5;
 168 
 169 /*
 170  * minimum lifespan of a prefetch block in clock ticks
 171  * (initialized in arc_init())
 172  */
 173 static int              arc_min_prefetch_lifespan;
 174 
 175 static int arc_dead;
 176 
 177 /*
 178  * The arc has filled available memory and has now warmed up.
 179  */
 180 static boolean_t arc_warm;
 181 
 182 /*
 183  * These tunables are for performance analysis.
 184  */
 185 uint64_t zfs_arc_max;
 186 uint64_t zfs_arc_min;
 187 uint64_t zfs_arc_meta_limit = 0;
 188 int zfs_arc_grow_retry = 0;
 189 int zfs_arc_shrink_shift = 0;
 190 int zfs_arc_p_min_shift = 0;
 191 int zfs_disable_dup_eviction = 0;
 192 
 193 /*
 194  * Note that buffers can be in one of 6 states:
 195  *      ARC_anon        - anonymous (discussed below)
 196  *      ARC_mru         - recently used, currently cached
 197  *      ARC_mru_ghost   - recentely used, no longer in cache
 198  *      ARC_mfu         - frequently used, currently cached
 199  *      ARC_mfu_ghost   - frequently used, no longer in cache
 200  *      ARC_l2c_only    - exists in L2ARC but not other states
 201  * When there are no active references to the buffer, they are
 202  * are linked onto a list in one of these arc states.  These are
 203  * the only buffers that can be evicted or deleted.  Within each
 204  * state there are multiple lists, one for meta-data and one for
 205  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
 206  * etc.) is tracked separately so that it can be managed more
 207  * explicitly: favored over data, limited explicitly.
 208  *
 209  * Anonymous buffers are buffers that are not associated with
 210  * a DVA.  These are buffers that hold dirty block copies
 211  * before they are written to stable storage.  By definition,
 212  * they are "ref'd" and are considered part of arc_mru
 213  * that cannot be freed.  Generally, they will aquire a DVA
 214  * as they are written and migrate onto the arc_mru list.
 215  *
 216  * The ARC_l2c_only state is for buffers that are in the second
 217  * level ARC but no longer in any of the ARC_m* lists.  The second
 218  * level ARC itself may also contain buffers that are in any of
 219  * the ARC_m* states - meaning that a buffer can exist in two
 220  * places.  The reason for the ARC_l2c_only state is to keep the
 221  * buffer header in the hash table, so that reads that hit the
 222  * second level ARC benefit from these fast lookups.
 223  */
 224 
 225 typedef struct arc_state {
 226         list_t  arcs_list[ARC_BUFC_NUMTYPES];   /* list of evictable buffers */
 227         uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */
 228         uint64_t arcs_size;     /* total amount of data in this state */
 229         kmutex_t arcs_mtx;
 230 } arc_state_t;
 231 
 232 /* The 6 states: */
 233 static arc_state_t ARC_anon;
 234 static arc_state_t ARC_mru;
 235 static arc_state_t ARC_mru_ghost;
 236 static arc_state_t ARC_mfu;
 237 static arc_state_t ARC_mfu_ghost;
 238 static arc_state_t ARC_l2c_only;
 239 
 240 typedef struct arc_stats {
 241         kstat_named_t arcstat_hits;
 242         kstat_named_t arcstat_misses;
 243         kstat_named_t arcstat_demand_data_hits;
 244         kstat_named_t arcstat_demand_data_misses;
 245         kstat_named_t arcstat_demand_metadata_hits;
 246         kstat_named_t arcstat_demand_metadata_misses;
 247         kstat_named_t arcstat_prefetch_data_hits;
 248         kstat_named_t arcstat_prefetch_data_misses;
 249         kstat_named_t arcstat_prefetch_metadata_hits;
 250         kstat_named_t arcstat_prefetch_metadata_misses;
 251         kstat_named_t arcstat_mru_hits;
 252         kstat_named_t arcstat_mru_ghost_hits;
 253         kstat_named_t arcstat_mfu_hits;
 254         kstat_named_t arcstat_mfu_ghost_hits;
 255         kstat_named_t arcstat_deleted;
 256         kstat_named_t arcstat_recycle_miss;
 257         /*
 258          * Number of buffers that could not be evicted because the hash lock
 259          * was held by another thread.  The lock may not necessarily be held
 260          * by something using the same buffer, since hash locks are shared
 261          * by multiple buffers.
 262          */
 263         kstat_named_t arcstat_mutex_miss;
 264         /*
 265          * Number of buffers skipped because they have I/O in progress, are
 266          * indrect prefetch buffers that have not lived long enough, or are
 267          * not from the spa we're trying to evict from.
 268          */
 269         kstat_named_t arcstat_evict_skip;
 270         kstat_named_t arcstat_evict_l2_cached;
 271         kstat_named_t arcstat_evict_l2_eligible;
 272         kstat_named_t arcstat_evict_l2_ineligible;
 273         kstat_named_t arcstat_hash_elements;
 274         kstat_named_t arcstat_hash_elements_max;
 275         kstat_named_t arcstat_hash_collisions;
 276         kstat_named_t arcstat_hash_chains;
 277         kstat_named_t arcstat_hash_chain_max;
 278         kstat_named_t arcstat_p;
 279         kstat_named_t arcstat_c;
 280         kstat_named_t arcstat_c_min;
 281         kstat_named_t arcstat_c_max;
 282         kstat_named_t arcstat_size;
 283         kstat_named_t arcstat_hdr_size;
 284         kstat_named_t arcstat_data_size;
 285         kstat_named_t arcstat_other_size;
 286         kstat_named_t arcstat_l2_hits;
 287         kstat_named_t arcstat_l2_misses;
 288         kstat_named_t arcstat_l2_feeds;
 289         kstat_named_t arcstat_l2_rw_clash;
 290         kstat_named_t arcstat_l2_read_bytes;
 291         kstat_named_t arcstat_l2_write_bytes;
 292         kstat_named_t arcstat_l2_writes_sent;
 293         kstat_named_t arcstat_l2_writes_done;
 294         kstat_named_t arcstat_l2_writes_error;
 295         kstat_named_t arcstat_l2_writes_hdr_miss;
 296         kstat_named_t arcstat_l2_evict_lock_retry;
 297         kstat_named_t arcstat_l2_evict_reading;
 298         kstat_named_t arcstat_l2_free_on_write;
 299         kstat_named_t arcstat_l2_abort_lowmem;
 300         kstat_named_t arcstat_l2_cksum_bad;
 301         kstat_named_t arcstat_l2_io_error;
 302         kstat_named_t arcstat_l2_size;
 303         kstat_named_t arcstat_l2_hdr_size;
 304         kstat_named_t arcstat_memory_throttle_count;
 305         kstat_named_t arcstat_duplicate_buffers;
 306         kstat_named_t arcstat_duplicate_buffers_size;
 307         kstat_named_t arcstat_duplicate_reads;
 308         kstat_named_t arcstat_meta_used;
 309         kstat_named_t arcstat_meta_limit;
 310         kstat_named_t arcstat_meta_max;
 311 } arc_stats_t;
 312 
 313 static arc_stats_t arc_stats = {
 314         { "hits",                       KSTAT_DATA_UINT64 },
 315         { "misses",                     KSTAT_DATA_UINT64 },
 316         { "demand_data_hits",           KSTAT_DATA_UINT64 },
 317         { "demand_data_misses",         KSTAT_DATA_UINT64 },
 318         { "demand_metadata_hits",       KSTAT_DATA_UINT64 },
 319         { "demand_metadata_misses",     KSTAT_DATA_UINT64 },
 320         { "prefetch_data_hits",         KSTAT_DATA_UINT64 },
 321         { "prefetch_data_misses",       KSTAT_DATA_UINT64 },
 322         { "prefetch_metadata_hits",     KSTAT_DATA_UINT64 },
 323         { "prefetch_metadata_misses",   KSTAT_DATA_UINT64 },
 324         { "mru_hits",                   KSTAT_DATA_UINT64 },
 325         { "mru_ghost_hits",             KSTAT_DATA_UINT64 },
 326         { "mfu_hits",                   KSTAT_DATA_UINT64 },
 327         { "mfu_ghost_hits",             KSTAT_DATA_UINT64 },
 328         { "deleted",                    KSTAT_DATA_UINT64 },
 329         { "recycle_miss",               KSTAT_DATA_UINT64 },
 330         { "mutex_miss",                 KSTAT_DATA_UINT64 },
 331         { "evict_skip",                 KSTAT_DATA_UINT64 },
 332         { "evict_l2_cached",            KSTAT_DATA_UINT64 },
 333         { "evict_l2_eligible",          KSTAT_DATA_UINT64 },
 334         { "evict_l2_ineligible",        KSTAT_DATA_UINT64 },
 335         { "hash_elements",              KSTAT_DATA_UINT64 },
 336         { "hash_elements_max",          KSTAT_DATA_UINT64 },
 337         { "hash_collisions",            KSTAT_DATA_UINT64 },
 338         { "hash_chains",                KSTAT_DATA_UINT64 },
 339         { "hash_chain_max",             KSTAT_DATA_UINT64 },
 340         { "p",                          KSTAT_DATA_UINT64 },
 341         { "c",                          KSTAT_DATA_UINT64 },
 342         { "c_min",                      KSTAT_DATA_UINT64 },
 343         { "c_max",                      KSTAT_DATA_UINT64 },
 344         { "size",                       KSTAT_DATA_UINT64 },
 345         { "hdr_size",                   KSTAT_DATA_UINT64 },
 346         { "data_size",                  KSTAT_DATA_UINT64 },
 347         { "other_size",                 KSTAT_DATA_UINT64 },
 348         { "l2_hits",                    KSTAT_DATA_UINT64 },
 349         { "l2_misses",                  KSTAT_DATA_UINT64 },
 350         { "l2_feeds",                   KSTAT_DATA_UINT64 },
 351         { "l2_rw_clash",                KSTAT_DATA_UINT64 },
 352         { "l2_read_bytes",              KSTAT_DATA_UINT64 },
 353         { "l2_write_bytes",             KSTAT_DATA_UINT64 },
 354         { "l2_writes_sent",             KSTAT_DATA_UINT64 },
 355         { "l2_writes_done",             KSTAT_DATA_UINT64 },
 356         { "l2_writes_error",            KSTAT_DATA_UINT64 },
 357         { "l2_writes_hdr_miss",         KSTAT_DATA_UINT64 },
 358         { "l2_evict_lock_retry",        KSTAT_DATA_UINT64 },
 359         { "l2_evict_reading",           KSTAT_DATA_UINT64 },
 360         { "l2_free_on_write",           KSTAT_DATA_UINT64 },
 361         { "l2_abort_lowmem",            KSTAT_DATA_UINT64 },
 362         { "l2_cksum_bad",               KSTAT_DATA_UINT64 },
 363         { "l2_io_error",                KSTAT_DATA_UINT64 },
 364         { "l2_size",                    KSTAT_DATA_UINT64 },
 365         { "l2_hdr_size",                KSTAT_DATA_UINT64 },
 366         { "memory_throttle_count",      KSTAT_DATA_UINT64 },
 367         { "duplicate_buffers",          KSTAT_DATA_UINT64 },
 368         { "duplicate_buffers_size",     KSTAT_DATA_UINT64 },
 369         { "duplicate_reads",            KSTAT_DATA_UINT64 },
 370         { "arc_meta_used",              KSTAT_DATA_UINT64 },
 371         { "arc_meta_limit",             KSTAT_DATA_UINT64 },
 372         { "arc_meta_max",               KSTAT_DATA_UINT64 }
 373 };
 374 
 375 #define ARCSTAT(stat)   (arc_stats.stat.value.ui64)
 376 
 377 #define ARCSTAT_INCR(stat, val) \
 378         atomic_add_64(&arc_stats.stat.value.ui64, (val));
 379 
 380 #define ARCSTAT_BUMP(stat)      ARCSTAT_INCR(stat, 1)
 381 #define ARCSTAT_BUMPDOWN(stat)  ARCSTAT_INCR(stat, -1)
 382 
 383 #define ARCSTAT_MAX(stat, val) {                                        \
 384         uint64_t m;                                                     \
 385         while ((val) > (m = arc_stats.stat.value.ui64) &&            \
 386             (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))     \
 387                 continue;                                               \
 388 }
 389 
 390 #define ARCSTAT_MAXSTAT(stat) \
 391         ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
 392 
 393 /*
 394  * We define a macro to allow ARC hits/misses to be easily broken down by
 395  * two separate conditions, giving a total of four different subtypes for
 396  * each of hits and misses (so eight statistics total).
 397  */
 398 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
 399         if (cond1) {                                                    \
 400                 if (cond2) {                                            \
 401                         ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
 402                 } else {                                                \
 403                         ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
 404                 }                                                       \
 405         } else {                                                        \
 406                 if (cond2) {                                            \
 407                         ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
 408                 } else {                                                \
 409                         ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
 410                 }                                                       \
 411         }
 412 
 413 kstat_t                 *arc_ksp;
 414 static arc_state_t      *arc_anon;
 415 static arc_state_t      *arc_mru;
 416 static arc_state_t      *arc_mru_ghost;
 417 static arc_state_t      *arc_mfu;
 418 static arc_state_t      *arc_mfu_ghost;
 419 static arc_state_t      *arc_l2c_only;
 420 
 421 /*
 422  * There are several ARC variables that are critical to export as kstats --
 423  * but we don't want to have to grovel around in the kstat whenever we wish to
 424  * manipulate them.  For these variables, we therefore define them to be in
 425  * terms of the statistic variable.  This assures that we are not introducing
 426  * the possibility of inconsistency by having shadow copies of the variables,
 427  * while still allowing the code to be readable.
 428  */
 429 #define arc_size        ARCSTAT(arcstat_size)   /* actual total arc size */
 430 #define arc_p           ARCSTAT(arcstat_p)      /* target size of MRU */
 431 #define arc_c           ARCSTAT(arcstat_c)      /* target size of cache */
 432 #define arc_c_min       ARCSTAT(arcstat_c_min)  /* min target cache size */
 433 #define arc_c_max       ARCSTAT(arcstat_c_max)  /* max target cache size */
 434 #define arc_meta_limit  ARCSTAT(arcstat_meta_limit) /* max size for metadata */
 435 #define arc_meta_used   ARCSTAT(arcstat_meta_used) /* size of metadata */
 436 #define arc_meta_max    ARCSTAT(arcstat_meta_max) /* max size of metadata */
 437 
 438 static int              arc_no_grow;    /* Don't try to grow cache size */
 439 static uint64_t         arc_tempreserve;
 440 static uint64_t         arc_loaned_bytes;
 441 
 442 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
 443 
 444 typedef struct arc_callback arc_callback_t;
 445 
 446 struct arc_callback {
 447         void                    *acb_private;
 448         arc_done_func_t         *acb_done;
 449         arc_buf_t               *acb_buf;
 450         zio_t                   *acb_zio_dummy;
 451         arc_callback_t          *acb_next;
 452 };
 453 
 454 typedef struct arc_write_callback arc_write_callback_t;
 455 
 456 struct arc_write_callback {
 457         void            *awcb_private;
 458         arc_done_func_t *awcb_ready;
 459         arc_done_func_t *awcb_done;
 460         arc_buf_t       *awcb_buf;
 461 };
 462 
 463 struct arc_buf_hdr {
 464         /* protected by hash lock */
 465         dva_t                   b_dva;
 466         uint64_t                b_birth;
 467         uint64_t                b_cksum0;
 468 
 469         kmutex_t                b_freeze_lock;
 470         zio_cksum_t             *b_freeze_cksum;
 471         void                    *b_thawed;
 472 
 473         arc_buf_hdr_t           *b_hash_next;
 474         arc_buf_t               *b_buf;
 475         uint32_t                b_flags;
 476         uint32_t                b_datacnt;
 477 
 478         arc_callback_t          *b_acb;
 479         kcondvar_t              b_cv;
 480 
 481         /* immutable */
 482         arc_buf_contents_t      b_type;
 483         uint64_t                b_size;
 484         uint64_t                b_spa;
 485 
 486         /* protected by arc state mutex */
 487         arc_state_t             *b_state;
 488         list_node_t             b_arc_node;
 489 
 490         /* updated atomically */
 491         clock_t                 b_arc_access;
 492 
 493         /* self protecting */
 494         refcount_t              b_refcnt;
 495 
 496         l2arc_buf_hdr_t         *b_l2hdr;
 497         list_node_t             b_l2node;
 498 };
 499 
 500 static arc_buf_t *arc_eviction_list;
 501 static kmutex_t arc_eviction_mtx;
 502 static arc_buf_hdr_t arc_eviction_hdr;
 503 static void arc_get_data_buf(arc_buf_t *buf);
 504 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
 505 static int arc_evict_needed(arc_buf_contents_t type);
 506 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
 507 static void arc_buf_watch(arc_buf_t *buf);
 508 
 509 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
 510 
 511 #define GHOST_STATE(state)      \
 512         ((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||        \
 513         (state) == arc_l2c_only)
 514 
 515 /*
 516  * Private ARC flags.  These flags are private ARC only flags that will show up
 517  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
 518  * be passed in as arc_flags in things like arc_read.  However, these flags
 519  * should never be passed and should only be set by ARC code.  When adding new
 520  * public flags, make sure not to smash the private ones.
 521  */
 522 
 523 #define ARC_IN_HASH_TABLE       (1 << 9)  /* this buffer is hashed */
 524 #define ARC_IO_IN_PROGRESS      (1 << 10) /* I/O in progress for buf */
 525 #define ARC_IO_ERROR            (1 << 11) /* I/O failed for buf */
 526 #define ARC_FREED_IN_READ       (1 << 12) /* buf freed while in read */
 527 #define ARC_BUF_AVAILABLE       (1 << 13) /* block not in active use */
 528 #define ARC_INDIRECT            (1 << 14) /* this is an indirect block */
 529 #define ARC_FREE_IN_PROGRESS    (1 << 15) /* hdr about to be freed */
 530 #define ARC_L2_WRITING          (1 << 16) /* L2ARC write in progress */
 531 #define ARC_L2_EVICTED          (1 << 17) /* evicted during I/O */
 532 #define ARC_L2_WRITE_HEAD       (1 << 18) /* head of write list */
 533 
 534 #define HDR_IN_HASH_TABLE(hdr)  ((hdr)->b_flags & ARC_IN_HASH_TABLE)
 535 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS)
 536 #define HDR_IO_ERROR(hdr)       ((hdr)->b_flags & ARC_IO_ERROR)
 537 #define HDR_PREFETCH(hdr)       ((hdr)->b_flags & ARC_PREFETCH)
 538 #define HDR_FREED_IN_READ(hdr)  ((hdr)->b_flags & ARC_FREED_IN_READ)
 539 #define HDR_BUF_AVAILABLE(hdr)  ((hdr)->b_flags & ARC_BUF_AVAILABLE)
 540 #define HDR_FREE_IN_PROGRESS(hdr)       ((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
 541 #define HDR_L2CACHE(hdr)        ((hdr)->b_flags & ARC_L2CACHE)
 542 #define HDR_L2_READING(hdr)     ((hdr)->b_flags & ARC_IO_IN_PROGRESS &&  \
 543                                     (hdr)->b_l2hdr != NULL)
 544 #define HDR_L2_WRITING(hdr)     ((hdr)->b_flags & ARC_L2_WRITING)
 545 #define HDR_L2_EVICTED(hdr)     ((hdr)->b_flags & ARC_L2_EVICTED)
 546 #define HDR_L2_WRITE_HEAD(hdr)  ((hdr)->b_flags & ARC_L2_WRITE_HEAD)
 547 
 548 /*
 549  * Other sizes
 550  */
 551 
 552 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
 553 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
 554 
 555 /*
 556  * Hash table routines
 557  */
 558 
 559 #define HT_LOCK_PAD     64
 560 
 561 struct ht_lock {
 562         kmutex_t        ht_lock;
 563 #ifdef _KERNEL
 564         unsigned char   pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
 565 #endif
 566 };
 567 
 568 #define BUF_LOCKS 256
 569 typedef struct buf_hash_table {
 570         uint64_t ht_mask;
 571         arc_buf_hdr_t **ht_table;
 572         struct ht_lock ht_locks[BUF_LOCKS];
 573 } buf_hash_table_t;
 574 
 575 static buf_hash_table_t buf_hash_table;
 576 
 577 #define BUF_HASH_INDEX(spa, dva, birth) \
 578         (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
 579 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
 580 #define BUF_HASH_LOCK(idx)      (&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
 581 #define HDR_LOCK(hdr) \
 582         (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
 583 
 584 uint64_t zfs_crc64_table[256];
 585 
 586 /*
 587  * Level 2 ARC
 588  */
 589 
 590 #define L2ARC_WRITE_SIZE        (8 * 1024 * 1024)       /* initial write max */
 591 #define L2ARC_HEADROOM          2               /* num of writes */
 592 #define L2ARC_FEED_SECS         1               /* caching interval secs */
 593 #define L2ARC_FEED_MIN_MS       200             /* min caching interval ms */
 594 
 595 #define l2arc_writes_sent       ARCSTAT(arcstat_l2_writes_sent)
 596 #define l2arc_writes_done       ARCSTAT(arcstat_l2_writes_done)
 597 
 598 /*
 599  * L2ARC Performance Tunables
 600  */
 601 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;    /* default max write size */
 602 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;  /* extra write during warmup */
 603 uint64_t l2arc_headroom = L2ARC_HEADROOM;       /* number of dev writes */
 604 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;     /* interval seconds */
 605 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
 606 boolean_t l2arc_noprefetch = B_TRUE;            /* don't cache prefetch bufs */
 607 boolean_t l2arc_feed_again = B_TRUE;            /* turbo warmup */
 608 boolean_t l2arc_norw = B_TRUE;                  /* no reads during writes */
 609 
 610 /*
 611  * L2ARC Internals
 612  */
 613 typedef struct l2arc_dev {
 614         vdev_t                  *l2ad_vdev;     /* vdev */
 615         spa_t                   *l2ad_spa;      /* spa */
 616         uint64_t                l2ad_hand;      /* next write location */
 617         uint64_t                l2ad_write;     /* desired write size, bytes */
 618         uint64_t                l2ad_boost;     /* warmup write boost, bytes */
 619         uint64_t                l2ad_start;     /* first addr on device */
 620         uint64_t                l2ad_end;       /* last addr on device */
 621         uint64_t                l2ad_evict;     /* last addr eviction reached */
 622         boolean_t               l2ad_first;     /* first sweep through */
 623         boolean_t               l2ad_writing;   /* currently writing */
 624         list_t                  *l2ad_buflist;  /* buffer list */
 625         list_node_t             l2ad_node;      /* device list node */
 626 } l2arc_dev_t;
 627 
 628 static list_t L2ARC_dev_list;                   /* device list */
 629 static list_t *l2arc_dev_list;                  /* device list pointer */
 630 static kmutex_t l2arc_dev_mtx;                  /* device list mutex */
 631 static l2arc_dev_t *l2arc_dev_last;             /* last device used */
 632 static kmutex_t l2arc_buflist_mtx;              /* mutex for all buflists */
 633 static list_t L2ARC_free_on_write;              /* free after write buf list */
 634 static list_t *l2arc_free_on_write;             /* free after write list ptr */
 635 static kmutex_t l2arc_free_on_write_mtx;        /* mutex for list */
 636 static uint64_t l2arc_ndev;                     /* number of devices */
 637 
 638 typedef struct l2arc_read_callback {
 639         arc_buf_t       *l2rcb_buf;             /* read buffer */
 640         spa_t           *l2rcb_spa;             /* spa */
 641         blkptr_t        l2rcb_bp;               /* original blkptr */
 642         zbookmark_t     l2rcb_zb;               /* original bookmark */
 643         int             l2rcb_flags;            /* original flags */
 644 } l2arc_read_callback_t;
 645 
 646 typedef struct l2arc_write_callback {
 647         l2arc_dev_t     *l2wcb_dev;             /* device info */
 648         arc_buf_hdr_t   *l2wcb_head;            /* head of write buflist */
 649 } l2arc_write_callback_t;
 650 
 651 struct l2arc_buf_hdr {
 652         /* protected by arc_buf_hdr  mutex */
 653         l2arc_dev_t     *b_dev;                 /* L2ARC device */
 654         uint64_t        b_daddr;                /* disk address, offset byte */
 655 };
 656 
 657 typedef struct l2arc_data_free {
 658         /* protected by l2arc_free_on_write_mtx */
 659         void            *l2df_data;
 660         size_t          l2df_size;
 661         void            (*l2df_func)(void *, size_t);
 662         list_node_t     l2df_list_node;
 663 } l2arc_data_free_t;
 664 
 665 static kmutex_t l2arc_feed_thr_lock;
 666 static kcondvar_t l2arc_feed_thr_cv;
 667 static uint8_t l2arc_thread_exit;
 668 
 669 static void l2arc_read_done(zio_t *zio);
 670 static void l2arc_hdr_stat_add(void);
 671 static void l2arc_hdr_stat_remove(void);
 672 
 673 static uint64_t
 674 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 675 {
 676         uint8_t *vdva = (uint8_t *)dva;
 677         uint64_t crc = -1ULL;
 678         int i;
 679 
 680         ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
 681 
 682         for (i = 0; i < sizeof (dva_t); i++)
 683                 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
 684 
 685         crc ^= (spa>>8) ^ birth;
 686 
 687         return (crc);
 688 }
 689 
 690 #define BUF_EMPTY(buf)                                          \
 691         ((buf)->b_dva.dva_word[0] == 0 &&                    \
 692         (buf)->b_dva.dva_word[1] == 0 &&                     \
 693         (buf)->b_birth == 0)
 694 
 695 #define BUF_EQUAL(spa, dva, birth, buf)                         \
 696         ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&       \
 697         ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&       \
 698         ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
 699 
 700 static void
 701 buf_discard_identity(arc_buf_hdr_t *hdr)
 702 {
 703         hdr->b_dva.dva_word[0] = 0;
 704         hdr->b_dva.dva_word[1] = 0;
 705         hdr->b_birth = 0;
 706         hdr->b_cksum0 = 0;
 707 }
 708 
 709 static arc_buf_hdr_t *
 710 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
 711 {
 712         uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 713         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 714         arc_buf_hdr_t *buf;
 715 
 716         mutex_enter(hash_lock);
 717         for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
 718             buf = buf->b_hash_next) {
 719                 if (BUF_EQUAL(spa, dva, birth, buf)) {
 720                         *lockp = hash_lock;
 721                         return (buf);
 722                 }
 723         }
 724         mutex_exit(hash_lock);
 725         *lockp = NULL;
 726         return (NULL);
 727 }
 728 
 729 /*
 730  * Insert an entry into the hash table.  If there is already an element
 731  * equal to elem in the hash table, then the already existing element
 732  * will be returned and the new element will not be inserted.
 733  * Otherwise returns NULL.
 734  */
 735 static arc_buf_hdr_t *
 736 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
 737 {
 738         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 739         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 740         arc_buf_hdr_t *fbuf;
 741         uint32_t i;
 742 
 743         ASSERT(!HDR_IN_HASH_TABLE(buf));
 744         *lockp = hash_lock;
 745         mutex_enter(hash_lock);
 746         for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
 747             fbuf = fbuf->b_hash_next, i++) {
 748                 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
 749                         return (fbuf);
 750         }
 751 
 752         buf->b_hash_next = buf_hash_table.ht_table[idx];
 753         buf_hash_table.ht_table[idx] = buf;
 754         buf->b_flags |= ARC_IN_HASH_TABLE;
 755 
 756         /* collect some hash table performance data */
 757         if (i > 0) {
 758                 ARCSTAT_BUMP(arcstat_hash_collisions);
 759                 if (i == 1)
 760                         ARCSTAT_BUMP(arcstat_hash_chains);
 761 
 762                 ARCSTAT_MAX(arcstat_hash_chain_max, i);
 763         }
 764 
 765         ARCSTAT_BUMP(arcstat_hash_elements);
 766         ARCSTAT_MAXSTAT(arcstat_hash_elements);
 767 
 768         return (NULL);
 769 }
 770 
 771 static void
 772 buf_hash_remove(arc_buf_hdr_t *buf)
 773 {
 774         arc_buf_hdr_t *fbuf, **bufp;
 775         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 776 
 777         ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
 778         ASSERT(HDR_IN_HASH_TABLE(buf));
 779 
 780         bufp = &buf_hash_table.ht_table[idx];
 781         while ((fbuf = *bufp) != buf) {
 782                 ASSERT(fbuf != NULL);
 783                 bufp = &fbuf->b_hash_next;
 784         }
 785         *bufp = buf->b_hash_next;
 786         buf->b_hash_next = NULL;
 787         buf->b_flags &= ~ARC_IN_HASH_TABLE;
 788 
 789         /* collect some hash table performance data */
 790         ARCSTAT_BUMPDOWN(arcstat_hash_elements);
 791 
 792         if (buf_hash_table.ht_table[idx] &&
 793             buf_hash_table.ht_table[idx]->b_hash_next == NULL)
 794                 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
 795 }
 796 
 797 /*
 798  * Global data structures and functions for the buf kmem cache.
 799  */
 800 static kmem_cache_t *hdr_cache;
 801 static kmem_cache_t *buf_cache;
 802 
 803 static void
 804 buf_fini(void)
 805 {
 806         int i;
 807 
 808         kmem_free(buf_hash_table.ht_table,
 809             (buf_hash_table.ht_mask + 1) * sizeof (void *));
 810         for (i = 0; i < BUF_LOCKS; i++)
 811                 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
 812         kmem_cache_destroy(hdr_cache);
 813         kmem_cache_destroy(buf_cache);
 814 }
 815 
 816 /*
 817  * Constructor callback - called when the cache is empty
 818  * and a new buf is requested.
 819  */
 820 /* ARGSUSED */
 821 static int
 822 hdr_cons(void *vbuf, void *unused, int kmflag)
 823 {
 824         arc_buf_hdr_t *buf = vbuf;
 825 
 826         bzero(buf, sizeof (arc_buf_hdr_t));
 827         refcount_create(&buf->b_refcnt);
 828         cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
 829         mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
 830         arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 831 
 832         return (0);
 833 }
 834 
 835 /* ARGSUSED */
 836 static int
 837 buf_cons(void *vbuf, void *unused, int kmflag)
 838 {
 839         arc_buf_t *buf = vbuf;
 840 
 841         bzero(buf, sizeof (arc_buf_t));
 842         mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
 843         arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 844 
 845         return (0);
 846 }
 847 
 848 /*
 849  * Destructor callback - called when a cached buf is
 850  * no longer required.
 851  */
 852 /* ARGSUSED */
 853 static void
 854 hdr_dest(void *vbuf, void *unused)
 855 {
 856         arc_buf_hdr_t *buf = vbuf;
 857 
 858         ASSERT(BUF_EMPTY(buf));
 859         refcount_destroy(&buf->b_refcnt);
 860         cv_destroy(&buf->b_cv);
 861         mutex_destroy(&buf->b_freeze_lock);
 862         arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 863 }
 864 
 865 /* ARGSUSED */
 866 static void
 867 buf_dest(void *vbuf, void *unused)
 868 {
 869         arc_buf_t *buf = vbuf;
 870 
 871         mutex_destroy(&buf->b_evict_lock);
 872         arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 873 }
 874 
 875 /*
 876  * Reclaim callback -- invoked when memory is low.
 877  */
 878 /* ARGSUSED */
 879 static void
 880 hdr_recl(void *unused)
 881 {
 882         dprintf("hdr_recl called\n");
 883         /*
 884          * umem calls the reclaim func when we destroy the buf cache,
 885          * which is after we do arc_fini().
 886          */
 887         if (!arc_dead)
 888                 cv_signal(&arc_reclaim_thr_cv);
 889 }
 890 
 891 static void
 892 buf_init(void)
 893 {
 894         uint64_t *ct;
 895         uint64_t hsize = 1ULL << 12;
 896         int i, j;
 897 
 898         /*
 899          * The hash table is big enough to fill all of physical memory
 900          * with an average 64K block size.  The table will take up
 901          * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
 902          */
 903         while (hsize * 65536 < physmem * PAGESIZE)
 904                 hsize <<= 1;
 905 retry:
 906         buf_hash_table.ht_mask = hsize - 1;
 907         buf_hash_table.ht_table =
 908             kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
 909         if (buf_hash_table.ht_table == NULL) {
 910                 ASSERT(hsize > (1ULL << 8));
 911                 hsize >>= 1;
 912                 goto retry;
 913         }
 914 
 915         hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
 916             0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
 917         buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 918             0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 919 
 920         for (i = 0; i < 256; i++)
 921                 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
 922                         *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
 923 
 924         for (i = 0; i < BUF_LOCKS; i++) {
 925                 mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
 926                     NULL, MUTEX_DEFAULT, NULL);
 927         }
 928 }
 929 
 930 #define ARC_MINTIME     (hz>>4) /* 62 ms */
 931 
 932 static void
 933 arc_cksum_verify(arc_buf_t *buf)
 934 {
 935         zio_cksum_t zc;
 936 
 937         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 938                 return;
 939 
 940         mutex_enter(&buf->b_hdr->b_freeze_lock);
 941         if (buf->b_hdr->b_freeze_cksum == NULL ||
 942             (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
 943                 mutex_exit(&buf->b_hdr->b_freeze_lock);
 944                 return;
 945         }
 946         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 947         if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 948                 panic("buffer modified while frozen!");
 949         mutex_exit(&buf->b_hdr->b_freeze_lock);
 950 }
 951 
 952 static int
 953 arc_cksum_equal(arc_buf_t *buf)
 954 {
 955         zio_cksum_t zc;
 956         int equal;
 957 
 958         mutex_enter(&buf->b_hdr->b_freeze_lock);
 959         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 960         equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
 961         mutex_exit(&buf->b_hdr->b_freeze_lock);
 962 
 963         return (equal);
 964 }
 965 
 966 static void
 967 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
 968 {
 969         if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
 970                 return;
 971 
 972         mutex_enter(&buf->b_hdr->b_freeze_lock);
 973         if (buf->b_hdr->b_freeze_cksum != NULL) {
 974                 mutex_exit(&buf->b_hdr->b_freeze_lock);
 975                 return;
 976         }
 977         buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
 978         fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
 979             buf->b_hdr->b_freeze_cksum);
 980         mutex_exit(&buf->b_hdr->b_freeze_lock);
 981         arc_buf_watch(buf);
 982 }
 983 
 984 #ifndef _KERNEL
 985 typedef struct procctl {
 986         long cmd;
 987         prwatch_t prwatch;
 988 } procctl_t;
 989 #endif
 990 
 991 /* ARGSUSED */
 992 static void
 993 arc_buf_unwatch(arc_buf_t *buf)
 994 {
 995 #ifndef _KERNEL
 996         if (arc_watch) {
 997                 int result;
 998                 procctl_t ctl;
 999                 ctl.cmd = PCWATCH;
1000                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1001                 ctl.prwatch.pr_size = 0;
1002                 ctl.prwatch.pr_wflags = 0;
1003                 result = write(arc_procfd, &ctl, sizeof (ctl));
1004                 ASSERT3U(result, ==, sizeof (ctl));
1005         }
1006 #endif
1007 }
1008 
1009 /* ARGSUSED */
1010 static void
1011 arc_buf_watch(arc_buf_t *buf)
1012 {
1013 #ifndef _KERNEL
1014         if (arc_watch) {
1015                 int result;
1016                 procctl_t ctl;
1017                 ctl.cmd = PCWATCH;
1018                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1019                 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1020                 ctl.prwatch.pr_wflags = WA_WRITE;
1021                 result = write(arc_procfd, &ctl, sizeof (ctl));
1022                 ASSERT3U(result, ==, sizeof (ctl));
1023         }
1024 #endif
1025 }
1026 
1027 void
1028 arc_buf_thaw(arc_buf_t *buf)
1029 {
1030         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1031                 if (buf->b_hdr->b_state != arc_anon)
1032                         panic("modifying non-anon buffer!");
1033                 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1034                         panic("modifying buffer while i/o in progress!");
1035                 arc_cksum_verify(buf);
1036         }
1037 
1038         mutex_enter(&buf->b_hdr->b_freeze_lock);
1039         if (buf->b_hdr->b_freeze_cksum != NULL) {
1040                 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1041                 buf->b_hdr->b_freeze_cksum = NULL;
1042         }
1043 
1044         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1045                 if (buf->b_hdr->b_thawed)
1046                         kmem_free(buf->b_hdr->b_thawed, 1);
1047                 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1048         }
1049 
1050         mutex_exit(&buf->b_hdr->b_freeze_lock);
1051 
1052         arc_buf_unwatch(buf);
1053 }
1054 
1055 void
1056 arc_buf_freeze(arc_buf_t *buf)
1057 {
1058         kmutex_t *hash_lock;
1059 
1060         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1061                 return;
1062 
1063         hash_lock = HDR_LOCK(buf->b_hdr);
1064         mutex_enter(hash_lock);
1065 
1066         ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1067             buf->b_hdr->b_state == arc_anon);
1068         arc_cksum_compute(buf, B_FALSE);
1069         mutex_exit(hash_lock);
1070 
1071 }
1072 
1073 static void
1074 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1075 {
1076         ASSERT(MUTEX_HELD(hash_lock));
1077 
1078         if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
1079             (ab->b_state != arc_anon)) {
1080                 uint64_t delta = ab->b_size * ab->b_datacnt;
1081                 list_t *list = &ab->b_state->arcs_list[ab->b_type];
1082                 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
1083 
1084                 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
1085                 mutex_enter(&ab->b_state->arcs_mtx);
1086                 ASSERT(list_link_active(&ab->b_arc_node));
1087                 list_remove(list, ab);
1088                 if (GHOST_STATE(ab->b_state)) {
1089                         ASSERT0(ab->b_datacnt);
1090                         ASSERT3P(ab->b_buf, ==, NULL);
1091                         delta = ab->b_size;
1092                 }
1093                 ASSERT(delta > 0);
1094                 ASSERT3U(*size, >=, delta);
1095                 atomic_add_64(size, -delta);
1096                 mutex_exit(&ab->b_state->arcs_mtx);
1097                 /* remove the prefetch flag if we get a reference */
1098                 if (ab->b_flags & ARC_PREFETCH)
1099                         ab->b_flags &= ~ARC_PREFETCH;
1100         }
1101 }
1102 
1103 static int
1104 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1105 {
1106         int cnt;
1107         arc_state_t *state = ab->b_state;
1108 
1109         ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1110         ASSERT(!GHOST_STATE(state));
1111 
1112         if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
1113             (state != arc_anon)) {
1114                 uint64_t *size = &state->arcs_lsize[ab->b_type];
1115 
1116                 ASSERT(!MUTEX_HELD(&state->arcs_mtx));
1117                 mutex_enter(&state->arcs_mtx);
1118                 ASSERT(!list_link_active(&ab->b_arc_node));
1119                 list_insert_head(&state->arcs_list[ab->b_type], ab);
1120                 ASSERT(ab->b_datacnt > 0);
1121                 atomic_add_64(size, ab->b_size * ab->b_datacnt);
1122                 mutex_exit(&state->arcs_mtx);
1123         }
1124         return (cnt);
1125 }
1126 
1127 /*
1128  * Move the supplied buffer to the indicated state.  The mutex
1129  * for the buffer must be held by the caller.
1130  */
1131 static void
1132 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1133 {
1134         arc_state_t *old_state = ab->b_state;
1135         int64_t refcnt = refcount_count(&ab->b_refcnt);
1136         uint64_t from_delta, to_delta;
1137 
1138         ASSERT(MUTEX_HELD(hash_lock));
1139         ASSERT(new_state != old_state);
1140         ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1141         ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1142         ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1143 
1144         from_delta = to_delta = ab->b_datacnt * ab->b_size;
1145 
1146         /*
1147          * If this buffer is evictable, transfer it from the
1148          * old state list to the new state list.
1149          */
1150         if (refcnt == 0) {
1151                 if (old_state != arc_anon) {
1152                         int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
1153                         uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1154 
1155                         if (use_mutex)
1156                                 mutex_enter(&old_state->arcs_mtx);
1157 
1158                         ASSERT(list_link_active(&ab->b_arc_node));
1159                         list_remove(&old_state->arcs_list[ab->b_type], ab);
1160 
1161                         /*
1162                          * If prefetching out of the ghost cache,
1163                          * we will have a non-zero datacnt.
1164                          */
1165                         if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
1166                                 /* ghost elements have a ghost size */
1167                                 ASSERT(ab->b_buf == NULL);
1168                                 from_delta = ab->b_size;
1169                         }
1170                         ASSERT3U(*size, >=, from_delta);
1171                         atomic_add_64(size, -from_delta);
1172 
1173                         if (use_mutex)
1174                                 mutex_exit(&old_state->arcs_mtx);
1175                 }
1176                 if (new_state != arc_anon) {
1177                         int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
1178                         uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1179 
1180                         if (use_mutex)
1181                                 mutex_enter(&new_state->arcs_mtx);
1182 
1183                         list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1184 
1185                         /* ghost elements have a ghost size */
1186                         if (GHOST_STATE(new_state)) {
1187                                 ASSERT(ab->b_datacnt == 0);
1188                                 ASSERT(ab->b_buf == NULL);
1189                                 to_delta = ab->b_size;
1190                         }
1191                         atomic_add_64(size, to_delta);
1192 
1193                         if (use_mutex)
1194                                 mutex_exit(&new_state->arcs_mtx);
1195                 }
1196         }
1197 
1198         ASSERT(!BUF_EMPTY(ab));
1199         if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
1200                 buf_hash_remove(ab);
1201 
1202         /* adjust state sizes */
1203         if (to_delta)
1204                 atomic_add_64(&new_state->arcs_size, to_delta);
1205         if (from_delta) {
1206                 ASSERT3U(old_state->arcs_size, >=, from_delta);
1207                 atomic_add_64(&old_state->arcs_size, -from_delta);
1208         }
1209         ab->b_state = new_state;
1210 
1211         /* adjust l2arc hdr stats */
1212         if (new_state == arc_l2c_only)
1213                 l2arc_hdr_stat_add();
1214         else if (old_state == arc_l2c_only)
1215                 l2arc_hdr_stat_remove();
1216 }
1217 
1218 void
1219 arc_space_consume(uint64_t space, arc_space_type_t type)
1220 {
1221         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1222 
1223         switch (type) {
1224         case ARC_SPACE_DATA:
1225                 ARCSTAT_INCR(arcstat_data_size, space);
1226                 break;
1227         case ARC_SPACE_OTHER:
1228                 ARCSTAT_INCR(arcstat_other_size, space);
1229                 break;
1230         case ARC_SPACE_HDRS:
1231                 ARCSTAT_INCR(arcstat_hdr_size, space);
1232                 break;
1233         case ARC_SPACE_L2HDRS:
1234                 ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1235                 break;
1236         }
1237 
1238         ARCSTAT_INCR(arcstat_meta_used, space);
1239         atomic_add_64(&arc_size, space);
1240 }
1241 
1242 void
1243 arc_space_return(uint64_t space, arc_space_type_t type)
1244 {
1245         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1246 
1247         switch (type) {
1248         case ARC_SPACE_DATA:
1249                 ARCSTAT_INCR(arcstat_data_size, -space);
1250                 break;
1251         case ARC_SPACE_OTHER:
1252                 ARCSTAT_INCR(arcstat_other_size, -space);
1253                 break;
1254         case ARC_SPACE_HDRS:
1255                 ARCSTAT_INCR(arcstat_hdr_size, -space);
1256                 break;
1257         case ARC_SPACE_L2HDRS:
1258                 ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1259                 break;
1260         }
1261 
1262         ASSERT(arc_meta_used >= space);
1263         if (arc_meta_max < arc_meta_used)
1264                 arc_meta_max = arc_meta_used;
1265         ARCSTAT_INCR(arcstat_meta_used, -space);
1266         ASSERT(arc_size >= space);
1267         atomic_add_64(&arc_size, -space);
1268 }
1269 
1270 void *
1271 arc_data_buf_alloc(uint64_t size)
1272 {
1273         if (arc_evict_needed(ARC_BUFC_DATA))
1274                 cv_signal(&arc_reclaim_thr_cv);
1275         atomic_add_64(&arc_size, size);
1276         return (zio_data_buf_alloc(size));
1277 }
1278 
1279 void
1280 arc_data_buf_free(void *buf, uint64_t size)
1281 {
1282         zio_data_buf_free(buf, size);
1283         ASSERT(arc_size >= size);
1284         atomic_add_64(&arc_size, -size);
1285 }
1286 
1287 arc_buf_t *
1288 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1289 {
1290         arc_buf_hdr_t *hdr;
1291         arc_buf_t *buf;
1292 
1293         ASSERT3U(size, >, 0);
1294         hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1295         ASSERT(BUF_EMPTY(hdr));
1296         hdr->b_size = size;
1297         hdr->b_type = type;
1298         hdr->b_spa = spa_load_guid(spa);
1299         hdr->b_state = arc_anon;
1300         hdr->b_arc_access = 0;
1301         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1302         buf->b_hdr = hdr;
1303         buf->b_data = NULL;
1304         buf->b_efunc = NULL;
1305         buf->b_private = NULL;
1306         buf->b_next = NULL;
1307         hdr->b_buf = buf;
1308         arc_get_data_buf(buf);
1309         hdr->b_datacnt = 1;
1310         hdr->b_flags = 0;
1311         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1312         (void) refcount_add(&hdr->b_refcnt, tag);
1313 
1314         return (buf);
1315 }
1316 
1317 static char *arc_onloan_tag = "onloan";
1318 
1319 /*
1320  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1321  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1322  * buffers must be returned to the arc before they can be used by the DMU or
1323  * freed.
1324  */
1325 arc_buf_t *
1326 arc_loan_buf(spa_t *spa, int size)
1327 {
1328         arc_buf_t *buf;
1329 
1330         buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1331 
1332         atomic_add_64(&arc_loaned_bytes, size);
1333         return (buf);
1334 }
1335 
1336 /*
1337  * Return a loaned arc buffer to the arc.
1338  */
1339 void
1340 arc_return_buf(arc_buf_t *buf, void *tag)
1341 {
1342         arc_buf_hdr_t *hdr = buf->b_hdr;
1343 
1344         ASSERT(buf->b_data != NULL);
1345         (void) refcount_add(&hdr->b_refcnt, tag);
1346         (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1347 
1348         atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1349 }
1350 
1351 /* Detach an arc_buf from a dbuf (tag) */
1352 void
1353 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1354 {
1355         arc_buf_hdr_t *hdr;
1356 
1357         ASSERT(buf->b_data != NULL);
1358         hdr = buf->b_hdr;
1359         (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1360         (void) refcount_remove(&hdr->b_refcnt, tag);
1361         buf->b_efunc = NULL;
1362         buf->b_private = NULL;
1363 
1364         atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1365 }
1366 
1367 static arc_buf_t *
1368 arc_buf_clone(arc_buf_t *from)
1369 {
1370         arc_buf_t *buf;
1371         arc_buf_hdr_t *hdr = from->b_hdr;
1372         uint64_t size = hdr->b_size;
1373 
1374         ASSERT(hdr->b_state != arc_anon);
1375 
1376         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1377         buf->b_hdr = hdr;
1378         buf->b_data = NULL;
1379         buf->b_efunc = NULL;
1380         buf->b_private = NULL;
1381         buf->b_next = hdr->b_buf;
1382         hdr->b_buf = buf;
1383         arc_get_data_buf(buf);
1384         bcopy(from->b_data, buf->b_data, size);
1385 
1386         /*
1387          * This buffer already exists in the arc so create a duplicate
1388          * copy for the caller.  If the buffer is associated with user data
1389          * then track the size and number of duplicates.  These stats will be
1390          * updated as duplicate buffers are created and destroyed.
1391          */
1392         if (hdr->b_type == ARC_BUFC_DATA) {
1393                 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1394                 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1395         }
1396         hdr->b_datacnt += 1;
1397         return (buf);
1398 }
1399 
1400 void
1401 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1402 {
1403         arc_buf_hdr_t *hdr;
1404         kmutex_t *hash_lock;
1405 
1406         /*
1407          * Check to see if this buffer is evicted.  Callers
1408          * must verify b_data != NULL to know if the add_ref
1409          * was successful.
1410          */
1411         mutex_enter(&buf->b_evict_lock);
1412         if (buf->b_data == NULL) {
1413                 mutex_exit(&buf->b_evict_lock);
1414                 return;
1415         }
1416         hash_lock = HDR_LOCK(buf->b_hdr);
1417         mutex_enter(hash_lock);
1418         hdr = buf->b_hdr;
1419         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1420         mutex_exit(&buf->b_evict_lock);
1421 
1422         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1423         add_reference(hdr, hash_lock, tag);
1424         DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1425         arc_access(hdr, hash_lock);
1426         mutex_exit(hash_lock);
1427         ARCSTAT_BUMP(arcstat_hits);
1428         ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1429             demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1430             data, metadata, hits);
1431 }
1432 
1433 /*
1434  * Free the arc data buffer.  If it is an l2arc write in progress,
1435  * the buffer is placed on l2arc_free_on_write to be freed later.
1436  */
1437 static void
1438 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1439 {
1440         arc_buf_hdr_t *hdr = buf->b_hdr;
1441 
1442         if (HDR_L2_WRITING(hdr)) {
1443                 l2arc_data_free_t *df;
1444                 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1445                 df->l2df_data = buf->b_data;
1446                 df->l2df_size = hdr->b_size;
1447                 df->l2df_func = free_func;
1448                 mutex_enter(&l2arc_free_on_write_mtx);
1449                 list_insert_head(l2arc_free_on_write, df);
1450                 mutex_exit(&l2arc_free_on_write_mtx);
1451                 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1452         } else {
1453                 free_func(buf->b_data, hdr->b_size);
1454         }
1455 }
1456 
1457 static void
1458 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1459 {
1460         arc_buf_t **bufp;
1461 
1462         /* free up data associated with the buf */
1463         if (buf->b_data) {
1464                 arc_state_t *state = buf->b_hdr->b_state;
1465                 uint64_t size = buf->b_hdr->b_size;
1466                 arc_buf_contents_t type = buf->b_hdr->b_type;
1467 
1468                 arc_cksum_verify(buf);
1469                 arc_buf_unwatch(buf);
1470 
1471                 if (!recycle) {
1472                         if (type == ARC_BUFC_METADATA) {
1473                                 arc_buf_data_free(buf, zio_buf_free);
1474                                 arc_space_return(size, ARC_SPACE_DATA);
1475                         } else {
1476                                 ASSERT(type == ARC_BUFC_DATA);
1477                                 arc_buf_data_free(buf, zio_data_buf_free);
1478                                 ARCSTAT_INCR(arcstat_data_size, -size);
1479                                 atomic_add_64(&arc_size, -size);
1480                         }
1481                 }
1482                 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1483                         uint64_t *cnt = &state->arcs_lsize[type];
1484 
1485                         ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
1486                         ASSERT(state != arc_anon);
1487 
1488                         ASSERT3U(*cnt, >=, size);
1489                         atomic_add_64(cnt, -size);
1490                 }
1491                 ASSERT3U(state->arcs_size, >=, size);
1492                 atomic_add_64(&state->arcs_size, -size);
1493                 buf->b_data = NULL;
1494 
1495                 /*
1496                  * If we're destroying a duplicate buffer make sure
1497                  * that the appropriate statistics are updated.
1498                  */
1499                 if (buf->b_hdr->b_datacnt > 1 &&
1500                     buf->b_hdr->b_type == ARC_BUFC_DATA) {
1501                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
1502                         ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
1503                 }
1504                 ASSERT(buf->b_hdr->b_datacnt > 0);
1505                 buf->b_hdr->b_datacnt -= 1;
1506         }
1507 
1508         /* only remove the buf if requested */
1509         if (!all)
1510                 return;
1511 
1512         /* remove the buf from the hdr list */
1513         for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1514                 continue;
1515         *bufp = buf->b_next;
1516         buf->b_next = NULL;
1517 
1518         ASSERT(buf->b_efunc == NULL);
1519 
1520         /* clean up the buf */
1521         buf->b_hdr = NULL;
1522         kmem_cache_free(buf_cache, buf);
1523 }
1524 
1525 static void
1526 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1527 {
1528         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1529         ASSERT3P(hdr->b_state, ==, arc_anon);
1530         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1531         l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1532 
1533         if (l2hdr != NULL) {
1534                 boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1535                 /*
1536                  * To prevent arc_free() and l2arc_evict() from
1537                  * attempting to free the same buffer at the same time,
1538                  * a FREE_IN_PROGRESS flag is given to arc_free() to
1539                  * give it priority.  l2arc_evict() can't destroy this
1540                  * header while we are waiting on l2arc_buflist_mtx.
1541                  *
1542                  * The hdr may be removed from l2ad_buflist before we
1543                  * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1544                  */
1545                 if (!buflist_held) {
1546                         mutex_enter(&l2arc_buflist_mtx);
1547                         l2hdr = hdr->b_l2hdr;
1548                 }
1549 
1550                 if (l2hdr != NULL) {
1551                         list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1552                         ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1553                         kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1554                         if (hdr->b_state == arc_l2c_only)
1555                                 l2arc_hdr_stat_remove();
1556                         hdr->b_l2hdr = NULL;
1557                 }
1558 
1559                 if (!buflist_held)
1560                         mutex_exit(&l2arc_buflist_mtx);
1561         }
1562 
1563         if (!BUF_EMPTY(hdr)) {
1564                 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1565                 buf_discard_identity(hdr);
1566         }
1567         while (hdr->b_buf) {
1568                 arc_buf_t *buf = hdr->b_buf;
1569 
1570                 if (buf->b_efunc) {
1571                         mutex_enter(&arc_eviction_mtx);
1572                         mutex_enter(&buf->b_evict_lock);
1573                         ASSERT(buf->b_hdr != NULL);
1574                         arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1575                         hdr->b_buf = buf->b_next;
1576                         buf->b_hdr = &arc_eviction_hdr;
1577                         buf->b_next = arc_eviction_list;
1578                         arc_eviction_list = buf;
1579                         mutex_exit(&buf->b_evict_lock);
1580                         mutex_exit(&arc_eviction_mtx);
1581                 } else {
1582                         arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1583                 }
1584         }
1585         if (hdr->b_freeze_cksum != NULL) {
1586                 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1587                 hdr->b_freeze_cksum = NULL;
1588         }
1589         if (hdr->b_thawed) {
1590                 kmem_free(hdr->b_thawed, 1);
1591                 hdr->b_thawed = NULL;
1592         }
1593 
1594         ASSERT(!list_link_active(&hdr->b_arc_node));
1595         ASSERT3P(hdr->b_hash_next, ==, NULL);
1596         ASSERT3P(hdr->b_acb, ==, NULL);
1597         kmem_cache_free(hdr_cache, hdr);
1598 }
1599 
1600 void
1601 arc_buf_free(arc_buf_t *buf, void *tag)
1602 {
1603         arc_buf_hdr_t *hdr = buf->b_hdr;
1604         int hashed = hdr->b_state != arc_anon;
1605 
1606         ASSERT(buf->b_efunc == NULL);
1607         ASSERT(buf->b_data != NULL);
1608 
1609         if (hashed) {
1610                 kmutex_t *hash_lock = HDR_LOCK(hdr);
1611 
1612                 mutex_enter(hash_lock);
1613                 hdr = buf->b_hdr;
1614                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1615 
1616                 (void) remove_reference(hdr, hash_lock, tag);
1617                 if (hdr->b_datacnt > 1) {
1618                         arc_buf_destroy(buf, FALSE, TRUE);
1619                 } else {
1620                         ASSERT(buf == hdr->b_buf);
1621                         ASSERT(buf->b_efunc == NULL);
1622                         hdr->b_flags |= ARC_BUF_AVAILABLE;
1623                 }
1624                 mutex_exit(hash_lock);
1625         } else if (HDR_IO_IN_PROGRESS(hdr)) {
1626                 int destroy_hdr;
1627                 /*
1628                  * We are in the middle of an async write.  Don't destroy
1629                  * this buffer unless the write completes before we finish
1630                  * decrementing the reference count.
1631                  */
1632                 mutex_enter(&arc_eviction_mtx);
1633                 (void) remove_reference(hdr, NULL, tag);
1634                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1635                 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1636                 mutex_exit(&arc_eviction_mtx);
1637                 if (destroy_hdr)
1638                         arc_hdr_destroy(hdr);
1639         } else {
1640                 if (remove_reference(hdr, NULL, tag) > 0)
1641                         arc_buf_destroy(buf, FALSE, TRUE);
1642                 else
1643                         arc_hdr_destroy(hdr);
1644         }
1645 }
1646 
1647 boolean_t
1648 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1649 {
1650         arc_buf_hdr_t *hdr = buf->b_hdr;
1651         kmutex_t *hash_lock = HDR_LOCK(hdr);
1652         boolean_t no_callback = (buf->b_efunc == NULL);
1653 
1654         if (hdr->b_state == arc_anon) {
1655                 ASSERT(hdr->b_datacnt == 1);
1656                 arc_buf_free(buf, tag);
1657                 return (no_callback);
1658         }
1659 
1660         mutex_enter(hash_lock);
1661         hdr = buf->b_hdr;
1662         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1663         ASSERT(hdr->b_state != arc_anon);
1664         ASSERT(buf->b_data != NULL);
1665 
1666         (void) remove_reference(hdr, hash_lock, tag);
1667         if (hdr->b_datacnt > 1) {
1668                 if (no_callback)
1669                         arc_buf_destroy(buf, FALSE, TRUE);
1670         } else if (no_callback) {
1671                 ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1672                 ASSERT(buf->b_efunc == NULL);
1673                 hdr->b_flags |= ARC_BUF_AVAILABLE;
1674         }
1675         ASSERT(no_callback || hdr->b_datacnt > 1 ||
1676             refcount_is_zero(&hdr->b_refcnt));
1677         mutex_exit(hash_lock);
1678         return (no_callback);
1679 }
1680 
1681 int
1682 arc_buf_size(arc_buf_t *buf)
1683 {
1684         return (buf->b_hdr->b_size);
1685 }
1686 
1687 /*
1688  * Called from the DMU to determine if the current buffer should be
1689  * evicted. In order to ensure proper locking, the eviction must be initiated
1690  * from the DMU. Return true if the buffer is associated with user data and
1691  * duplicate buffers still exist.
1692  */
1693 boolean_t
1694 arc_buf_eviction_needed(arc_buf_t *buf)
1695 {
1696         arc_buf_hdr_t *hdr;
1697         boolean_t evict_needed = B_FALSE;
1698 
1699         if (zfs_disable_dup_eviction)
1700                 return (B_FALSE);
1701 
1702         mutex_enter(&buf->b_evict_lock);
1703         hdr = buf->b_hdr;
1704         if (hdr == NULL) {
1705                 /*
1706                  * We are in arc_do_user_evicts(); let that function
1707                  * perform the eviction.
1708                  */
1709                 ASSERT(buf->b_data == NULL);
1710                 mutex_exit(&buf->b_evict_lock);
1711                 return (B_FALSE);
1712         } else if (buf->b_data == NULL) {
1713                 /*
1714                  * We have already been added to the arc eviction list;
1715                  * recommend eviction.
1716                  */
1717                 ASSERT3P(hdr, ==, &arc_eviction_hdr);
1718                 mutex_exit(&buf->b_evict_lock);
1719                 return (B_TRUE);
1720         }
1721 
1722         if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
1723                 evict_needed = B_TRUE;
1724 
1725         mutex_exit(&buf->b_evict_lock);
1726         return (evict_needed);
1727 }
1728 
1729 /*
1730  * Evict buffers from list until we've removed the specified number of
1731  * bytes.  Move the removed buffers to the appropriate evict state.
1732  * If the recycle flag is set, then attempt to "recycle" a buffer:
1733  * - look for a buffer to evict that is `bytes' long.
1734  * - return the data block from this buffer rather than freeing it.
1735  * This flag is used by callers that are trying to make space for a
1736  * new buffer in a full arc cache.
1737  *
1738  * This function makes a "best effort".  It skips over any buffers
1739  * it can't get a hash_lock on, and so may not catch all candidates.
1740  * It may also return without evicting as much space as requested.
1741  */
1742 static void *
1743 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1744     arc_buf_contents_t type)
1745 {
1746         arc_state_t *evicted_state;
1747         uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1748         arc_buf_hdr_t *ab, *ab_prev = NULL;
1749         list_t *list = &state->arcs_list[type];
1750         kmutex_t *hash_lock;
1751         boolean_t have_lock;
1752         void *stolen = NULL;
1753 
1754         ASSERT(state == arc_mru || state == arc_mfu);
1755 
1756         evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1757 
1758         mutex_enter(&state->arcs_mtx);
1759         mutex_enter(&evicted_state->arcs_mtx);
1760 
1761         for (ab = list_tail(list); ab; ab = ab_prev) {
1762                 ab_prev = list_prev(list, ab);
1763                 /* prefetch buffers have a minimum lifespan */
1764                 if (HDR_IO_IN_PROGRESS(ab) ||
1765                     (spa && ab->b_spa != spa) ||
1766                     (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1767                     ddi_get_lbolt() - ab->b_arc_access <
1768                     arc_min_prefetch_lifespan)) {
1769                         skipped++;
1770                         continue;
1771                 }
1772                 /* "lookahead" for better eviction candidate */
1773                 if (recycle && ab->b_size != bytes &&
1774                     ab_prev && ab_prev->b_size == bytes)
1775                         continue;
1776                 hash_lock = HDR_LOCK(ab);
1777                 have_lock = MUTEX_HELD(hash_lock);
1778                 if (have_lock || mutex_tryenter(hash_lock)) {
1779                         ASSERT0(refcount_count(&ab->b_refcnt));
1780                         ASSERT(ab->b_datacnt > 0);
1781                         while (ab->b_buf) {
1782                                 arc_buf_t *buf = ab->b_buf;
1783                                 if (!mutex_tryenter(&buf->b_evict_lock)) {
1784                                         missed += 1;
1785                                         break;
1786                                 }
1787                                 if (buf->b_data) {
1788                                         bytes_evicted += ab->b_size;
1789                                         if (recycle && ab->b_type == type &&
1790                                             ab->b_size == bytes &&
1791                                             !HDR_L2_WRITING(ab)) {
1792                                                 stolen = buf->b_data;
1793                                                 recycle = FALSE;
1794                                         }
1795                                 }
1796                                 if (buf->b_efunc) {
1797                                         mutex_enter(&arc_eviction_mtx);
1798                                         arc_buf_destroy(buf,
1799                                             buf->b_data == stolen, FALSE);
1800                                         ab->b_buf = buf->b_next;
1801                                         buf->b_hdr = &arc_eviction_hdr;
1802                                         buf->b_next = arc_eviction_list;
1803                                         arc_eviction_list = buf;
1804                                         mutex_exit(&arc_eviction_mtx);
1805                                         mutex_exit(&buf->b_evict_lock);
1806                                 } else {
1807                                         mutex_exit(&buf->b_evict_lock);
1808                                         arc_buf_destroy(buf,
1809                                             buf->b_data == stolen, TRUE);
1810                                 }
1811                         }
1812 
1813                         if (ab->b_l2hdr) {
1814                                 ARCSTAT_INCR(arcstat_evict_l2_cached,
1815                                     ab->b_size);
1816                         } else {
1817                                 if (l2arc_write_eligible(ab->b_spa, ab)) {
1818                                         ARCSTAT_INCR(arcstat_evict_l2_eligible,
1819                                             ab->b_size);
1820                                 } else {
1821                                         ARCSTAT_INCR(
1822                                             arcstat_evict_l2_ineligible,
1823                                             ab->b_size);
1824                                 }
1825                         }
1826 
1827                         if (ab->b_datacnt == 0) {
1828                                 arc_change_state(evicted_state, ab, hash_lock);
1829                                 ASSERT(HDR_IN_HASH_TABLE(ab));
1830                                 ab->b_flags |= ARC_IN_HASH_TABLE;
1831                                 ab->b_flags &= ~ARC_BUF_AVAILABLE;
1832                                 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
1833                         }
1834                         if (!have_lock)
1835                                 mutex_exit(hash_lock);
1836                         if (bytes >= 0 && bytes_evicted >= bytes)
1837                                 break;
1838                 } else {
1839                         missed += 1;
1840                 }
1841         }
1842 
1843         mutex_exit(&evicted_state->arcs_mtx);
1844         mutex_exit(&state->arcs_mtx);
1845 
1846         if (bytes_evicted < bytes)
1847                 dprintf("only evicted %lld bytes from %x",
1848                     (longlong_t)bytes_evicted, state);
1849 
1850         if (skipped)
1851                 ARCSTAT_INCR(arcstat_evict_skip, skipped);
1852 
1853         if (missed)
1854                 ARCSTAT_INCR(arcstat_mutex_miss, missed);
1855 
1856         /*
1857          * We have just evicted some data into the ghost state, make
1858          * sure we also adjust the ghost state size if necessary.
1859          */
1860         if (arc_no_grow &&
1861             arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1862                 int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1863                     arc_mru_ghost->arcs_size - arc_c;
1864 
1865                 if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1866                         int64_t todelete =
1867                             MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1868                         arc_evict_ghost(arc_mru_ghost, NULL, todelete);
1869                 } else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1870                         int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1871                             arc_mru_ghost->arcs_size +
1872                             arc_mfu_ghost->arcs_size - arc_c);
1873                         arc_evict_ghost(arc_mfu_ghost, NULL, todelete);
1874                 }
1875         }
1876 
1877         return (stolen);
1878 }
1879 
1880 /*
1881  * Remove buffers from list until we've removed the specified number of
1882  * bytes.  Destroy the buffers that are removed.
1883  */
1884 static void
1885 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1886 {
1887         arc_buf_hdr_t *ab, *ab_prev;
1888         arc_buf_hdr_t marker = { 0 };
1889         list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1890         kmutex_t *hash_lock;
1891         uint64_t bytes_deleted = 0;
1892         uint64_t bufs_skipped = 0;
1893 
1894         ASSERT(GHOST_STATE(state));
1895 top:
1896         mutex_enter(&state->arcs_mtx);
1897         for (ab = list_tail(list); ab; ab = ab_prev) {
1898                 ab_prev = list_prev(list, ab);
1899                 if (spa && ab->b_spa != spa)
1900                         continue;
1901 
1902                 /* ignore markers */
1903                 if (ab->b_spa == 0)
1904                         continue;
1905 
1906                 hash_lock = HDR_LOCK(ab);
1907                 /* caller may be trying to modify this buffer, skip it */
1908                 if (MUTEX_HELD(hash_lock))
1909                         continue;
1910                 if (mutex_tryenter(hash_lock)) {
1911                         ASSERT(!HDR_IO_IN_PROGRESS(ab));
1912                         ASSERT(ab->b_buf == NULL);
1913                         ARCSTAT_BUMP(arcstat_deleted);
1914                         bytes_deleted += ab->b_size;
1915 
1916                         if (ab->b_l2hdr != NULL) {
1917                                 /*
1918                                  * This buffer is cached on the 2nd Level ARC;
1919                                  * don't destroy the header.
1920                                  */
1921                                 arc_change_state(arc_l2c_only, ab, hash_lock);
1922                                 mutex_exit(hash_lock);
1923                         } else {
1924                                 arc_change_state(arc_anon, ab, hash_lock);
1925                                 mutex_exit(hash_lock);
1926                                 arc_hdr_destroy(ab);
1927                         }
1928 
1929                         DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1930                         if (bytes >= 0 && bytes_deleted >= bytes)
1931                                 break;
1932                 } else if (bytes < 0) {
1933                         /*
1934                          * Insert a list marker and then wait for the
1935                          * hash lock to become available. Once its
1936                          * available, restart from where we left off.
1937                          */
1938                         list_insert_after(list, ab, &marker);
1939                         mutex_exit(&state->arcs_mtx);
1940                         mutex_enter(hash_lock);
1941                         mutex_exit(hash_lock);
1942                         mutex_enter(&state->arcs_mtx);
1943                         ab_prev = list_prev(list, &marker);
1944                         list_remove(list, &marker);
1945                 } else
1946                         bufs_skipped += 1;
1947         }
1948         mutex_exit(&state->arcs_mtx);
1949 
1950         if (list == &state->arcs_list[ARC_BUFC_DATA] &&
1951             (bytes < 0 || bytes_deleted < bytes)) {
1952                 list = &state->arcs_list[ARC_BUFC_METADATA];
1953                 goto top;
1954         }
1955 
1956         if (bufs_skipped) {
1957                 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
1958                 ASSERT(bytes >= 0);
1959         }
1960 
1961         if (bytes_deleted < bytes)
1962                 dprintf("only deleted %lld bytes from %p",
1963                     (longlong_t)bytes_deleted, state);
1964 }
1965 
1966 static void
1967 arc_adjust(void)
1968 {
1969         int64_t adjustment, delta;
1970 
1971         /*
1972          * Adjust MRU size
1973          */
1974 
1975         adjustment = MIN((int64_t)(arc_size - arc_c),
1976             (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
1977             arc_p));
1978 
1979         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
1980                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
1981                 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
1982                 adjustment -= delta;
1983         }
1984 
1985         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
1986                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
1987                 (void) arc_evict(arc_mru, NULL, delta, FALSE,
1988                     ARC_BUFC_METADATA);
1989         }
1990 
1991         /*
1992          * Adjust MFU size
1993          */
1994 
1995         adjustment = arc_size - arc_c;
1996 
1997         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
1998                 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
1999                 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
2000                 adjustment -= delta;
2001         }
2002 
2003         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2004                 int64_t delta = MIN(adjustment,
2005                     arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
2006                 (void) arc_evict(arc_mfu, NULL, delta, FALSE,
2007                     ARC_BUFC_METADATA);
2008         }
2009 
2010         /*
2011          * Adjust ghost lists
2012          */
2013 
2014         adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2015 
2016         if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
2017                 delta = MIN(arc_mru_ghost->arcs_size, adjustment);
2018                 arc_evict_ghost(arc_mru_ghost, NULL, delta);
2019         }
2020 
2021         adjustment =
2022             arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2023 
2024         if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2025                 delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2026                 arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2027         }
2028 }
2029 
2030 static void
2031 arc_do_user_evicts(void)
2032 {
2033         mutex_enter(&arc_eviction_mtx);
2034         while (arc_eviction_list != NULL) {
2035                 arc_buf_t *buf = arc_eviction_list;
2036                 arc_eviction_list = buf->b_next;
2037                 mutex_enter(&buf->b_evict_lock);
2038                 buf->b_hdr = NULL;
2039                 mutex_exit(&buf->b_evict_lock);
2040                 mutex_exit(&arc_eviction_mtx);
2041 
2042                 if (buf->b_efunc != NULL)
2043                         VERIFY(buf->b_efunc(buf) == 0);
2044 
2045                 buf->b_efunc = NULL;
2046                 buf->b_private = NULL;
2047                 kmem_cache_free(buf_cache, buf);
2048                 mutex_enter(&arc_eviction_mtx);
2049         }
2050         mutex_exit(&arc_eviction_mtx);
2051 }
2052 
2053 /*
2054  * Flush all *evictable* data from the cache for the given spa.
2055  * NOTE: this will not touch "active" (i.e. referenced) data.
2056  */
2057 void
2058 arc_flush(spa_t *spa)
2059 {
2060         uint64_t guid = 0;
2061 
2062         if (spa)
2063                 guid = spa_load_guid(spa);
2064 
2065         while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
2066                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2067                 if (spa)
2068                         break;
2069         }
2070         while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
2071                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2072                 if (spa)
2073                         break;
2074         }
2075         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
2076                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2077                 if (spa)
2078                         break;
2079         }
2080         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
2081                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2082                 if (spa)
2083                         break;
2084         }
2085 
2086         arc_evict_ghost(arc_mru_ghost, guid, -1);
2087         arc_evict_ghost(arc_mfu_ghost, guid, -1);
2088 
2089         mutex_enter(&arc_reclaim_thr_lock);
2090         arc_do_user_evicts();
2091         mutex_exit(&arc_reclaim_thr_lock);
2092         ASSERT(spa || arc_eviction_list == NULL);
2093 }
2094 
2095 void
2096 arc_shrink(void)
2097 {
2098         if (arc_c > arc_c_min) {
2099                 uint64_t to_free;
2100 
2101 #ifdef _KERNEL
2102                 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
2103 #else
2104                 to_free = arc_c >> arc_shrink_shift;
2105 #endif
2106                 if (arc_c > arc_c_min + to_free)
2107                         atomic_add_64(&arc_c, -to_free);
2108                 else
2109                         arc_c = arc_c_min;
2110 
2111                 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
2112                 if (arc_c > arc_size)
2113                         arc_c = MAX(arc_size, arc_c_min);
2114                 if (arc_p > arc_c)
2115                         arc_p = (arc_c >> 1);
2116                 ASSERT(arc_c >= arc_c_min);
2117                 ASSERT((int64_t)arc_p >= 0);
2118         }
2119 
2120         if (arc_size > arc_c)
2121                 arc_adjust();
2122 }
2123 
2124 /*
2125  * Determine if the system is under memory pressure and is asking
2126  * to reclaim memory. A return value of 1 indicates that the system
2127  * is under memory pressure and that the arc should adjust accordingly.
2128  */
2129 static int
2130 arc_reclaim_needed(void)
2131 {
2132         uint64_t extra;
2133 
2134 #ifdef _KERNEL
2135 
2136         if (needfree)
2137                 return (1);
2138 
2139         /*
2140          * take 'desfree' extra pages, so we reclaim sooner, rather than later
2141          */
2142         extra = desfree;
2143 
2144         /*
2145          * check that we're out of range of the pageout scanner.  It starts to
2146          * schedule paging if freemem is less than lotsfree and needfree.
2147          * lotsfree is the high-water mark for pageout, and needfree is the
2148          * number of needed free pages.  We add extra pages here to make sure
2149          * the scanner doesn't start up while we're freeing memory.
2150          */
2151         if (freemem < lotsfree + needfree + extra)
2152                 return (1);
2153 
2154         /*
2155          * check to make sure that swapfs has enough space so that anon
2156          * reservations can still succeed. anon_resvmem() checks that the
2157          * availrmem is greater than swapfs_minfree, and the number of reserved
2158          * swap pages.  We also add a bit of extra here just to prevent
2159          * circumstances from getting really dire.
2160          */
2161         if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2162                 return (1);
2163 
2164 #if defined(__i386)
2165         /*
2166          * If we're on an i386 platform, it's possible that we'll exhaust the
2167          * kernel heap space before we ever run out of available physical
2168          * memory.  Most checks of the size of the heap_area compare against
2169          * tune.t_minarmem, which is the minimum available real memory that we
2170          * can have in the system.  However, this is generally fixed at 25 pages
2171          * which is so low that it's useless.  In this comparison, we seek to
2172          * calculate the total heap-size, and reclaim if more than 3/4ths of the
2173          * heap is allocated.  (Or, in the calculation, if less than 1/4th is
2174          * free)
2175          */
2176         if (vmem_size(heap_arena, VMEM_FREE) <
2177             (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
2178                 return (1);
2179 #endif
2180 
2181         /*
2182          * If zio data pages are being allocated out of a separate heap segment,
2183          * then enforce that the size of available vmem for this arena remains
2184          * above about 1/16th free.
2185          *
2186          * Note: The 1/16th arena free requirement was put in place
2187          * to aggressively evict memory from the arc in order to avoid
2188          * memory fragmentation issues.
2189          */
2190         if (zio_arena != NULL &&
2191             vmem_size(zio_arena, VMEM_FREE) <
2192             (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
2193                 return (1);
2194 #else
2195         if (spa_get_random(100) == 0)
2196                 return (1);
2197 #endif
2198         return (0);
2199 }
2200 
2201 static void
2202 arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2203 {
2204         size_t                  i;
2205         kmem_cache_t            *prev_cache = NULL;
2206         kmem_cache_t            *prev_data_cache = NULL;
2207         extern kmem_cache_t     *zio_buf_cache[];
2208         extern kmem_cache_t     *zio_data_buf_cache[];
2209 
2210 #ifdef _KERNEL
2211         if (arc_meta_used >= arc_meta_limit) {
2212                 /*
2213                  * We are exceeding our meta-data cache limit.
2214                  * Purge some DNLC entries to release holds on meta-data.
2215                  */
2216                 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
2217         }
2218 #if defined(__i386)
2219         /*
2220          * Reclaim unused memory from all kmem caches.
2221          */
2222         kmem_reap();
2223 #endif
2224 #endif
2225 
2226         /*
2227          * An aggressive reclamation will shrink the cache size as well as
2228          * reap free buffers from the arc kmem caches.
2229          */
2230         if (strat == ARC_RECLAIM_AGGR)
2231                 arc_shrink();
2232 
2233         for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2234                 if (zio_buf_cache[i] != prev_cache) {
2235                         prev_cache = zio_buf_cache[i];
2236                         kmem_cache_reap_now(zio_buf_cache[i]);
2237                 }
2238                 if (zio_data_buf_cache[i] != prev_data_cache) {
2239                         prev_data_cache = zio_data_buf_cache[i];
2240                         kmem_cache_reap_now(zio_data_buf_cache[i]);
2241                 }
2242         }
2243         kmem_cache_reap_now(buf_cache);
2244         kmem_cache_reap_now(hdr_cache);
2245 
2246         /*
2247          * Ask the vmem areana to reclaim unused memory from its
2248          * quantum caches.
2249          */
2250         if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
2251                 vmem_qcache_reap(zio_arena);
2252 }
2253 
2254 static void
2255 arc_reclaim_thread(void)
2256 {
2257         clock_t                 growtime = 0;
2258         arc_reclaim_strategy_t  last_reclaim = ARC_RECLAIM_CONS;
2259         callb_cpr_t             cpr;
2260 
2261         CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2262 
2263         mutex_enter(&arc_reclaim_thr_lock);
2264         while (arc_thread_exit == 0) {
2265                 if (arc_reclaim_needed()) {
2266 
2267                         if (arc_no_grow) {
2268                                 if (last_reclaim == ARC_RECLAIM_CONS) {
2269                                         last_reclaim = ARC_RECLAIM_AGGR;
2270                                 } else {
2271                                         last_reclaim = ARC_RECLAIM_CONS;
2272                                 }
2273                         } else {
2274                                 arc_no_grow = TRUE;
2275                                 last_reclaim = ARC_RECLAIM_AGGR;
2276                                 membar_producer();
2277                         }
2278 
2279                         /* reset the growth delay for every reclaim */
2280                         growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2281 
2282                         arc_kmem_reap_now(last_reclaim);
2283                         arc_warm = B_TRUE;
2284 
2285                 } else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
2286                         arc_no_grow = FALSE;
2287                 }
2288 
2289                 arc_adjust();
2290 
2291                 if (arc_eviction_list != NULL)
2292                         arc_do_user_evicts();
2293 
2294                 /* block until needed, or one second, whichever is shorter */
2295                 CALLB_CPR_SAFE_BEGIN(&cpr);
2296                 (void) cv_timedwait(&arc_reclaim_thr_cv,
2297                     &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2298                 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2299         }
2300 
2301         arc_thread_exit = 0;
2302         cv_broadcast(&arc_reclaim_thr_cv);
2303         CALLB_CPR_EXIT(&cpr);               /* drops arc_reclaim_thr_lock */
2304         thread_exit();
2305 }
2306 
2307 /*
2308  * Adapt arc info given the number of bytes we are trying to add and
2309  * the state that we are comming from.  This function is only called
2310  * when we are adding new content to the cache.
2311  */
2312 static void
2313 arc_adapt(int bytes, arc_state_t *state)
2314 {
2315         int mult;
2316         uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2317 
2318         if (state == arc_l2c_only)
2319                 return;
2320 
2321         ASSERT(bytes > 0);
2322         /*
2323          * Adapt the target size of the MRU list:
2324          *      - if we just hit in the MRU ghost list, then increase
2325          *        the target size of the MRU list.
2326          *      - if we just hit in the MFU ghost list, then increase
2327          *        the target size of the MFU list by decreasing the
2328          *        target size of the MRU list.
2329          */
2330         if (state == arc_mru_ghost) {
2331                 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
2332                     1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
2333                 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
2334 
2335                 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
2336         } else if (state == arc_mfu_ghost) {
2337                 uint64_t delta;
2338 
2339                 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
2340                     1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
2341                 mult = MIN(mult, 10);
2342 
2343                 delta = MIN(bytes * mult, arc_p);
2344                 arc_p = MAX(arc_p_min, arc_p - delta);
2345         }
2346         ASSERT((int64_t)arc_p >= 0);
2347 
2348         if (arc_reclaim_needed()) {
2349                 cv_signal(&arc_reclaim_thr_cv);
2350                 return;
2351         }
2352 
2353         if (arc_no_grow)
2354                 return;
2355 
2356         if (arc_c >= arc_c_max)
2357                 return;
2358 
2359         /*
2360          * If we're within (2 * maxblocksize) bytes of the target
2361          * cache size, increment the target cache size
2362          */
2363         if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
2364                 atomic_add_64(&arc_c, (int64_t)bytes);
2365                 if (arc_c > arc_c_max)
2366                         arc_c = arc_c_max;
2367                 else if (state == arc_anon)
2368                         atomic_add_64(&arc_p, (int64_t)bytes);
2369                 if (arc_p > arc_c)
2370                         arc_p = arc_c;
2371         }
2372         ASSERT((int64_t)arc_p >= 0);
2373 }
2374 
2375 /*
2376  * Check if the cache has reached its limits and eviction is required
2377  * prior to insert.
2378  */
2379 static int
2380 arc_evict_needed(arc_buf_contents_t type)
2381 {
2382         if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2383                 return (1);
2384 
2385         if (arc_reclaim_needed())
2386                 return (1);
2387 
2388         return (arc_size > arc_c);
2389 }
2390 
2391 /*
2392  * The buffer, supplied as the first argument, needs a data block.
2393  * So, if we are at cache max, determine which cache should be victimized.
2394  * We have the following cases:
2395  *
2396  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2397  * In this situation if we're out of space, but the resident size of the MFU is
2398  * under the limit, victimize the MFU cache to satisfy this insertion request.
2399  *
2400  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2401  * Here, we've used up all of the available space for the MRU, so we need to
2402  * evict from our own cache instead.  Evict from the set of resident MRU
2403  * entries.
2404  *
2405  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2406  * c minus p represents the MFU space in the cache, since p is the size of the
2407  * cache that is dedicated to the MRU.  In this situation there's still space on
2408  * the MFU side, so the MRU side needs to be victimized.
2409  *
2410  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2411  * MFU's resident set is consuming more space than it has been allotted.  In
2412  * this situation, we must victimize our own cache, the MFU, for this insertion.
2413  */
2414 static void
2415 arc_get_data_buf(arc_buf_t *buf)
2416 {
2417         arc_state_t             *state = buf->b_hdr->b_state;
2418         uint64_t                size = buf->b_hdr->b_size;
2419         arc_buf_contents_t      type = buf->b_hdr->b_type;
2420 
2421         arc_adapt(size, state);
2422 
2423         /*
2424          * We have not yet reached cache maximum size,
2425          * just allocate a new buffer.
2426          */
2427         if (!arc_evict_needed(type)) {
2428                 if (type == ARC_BUFC_METADATA) {
2429                         buf->b_data = zio_buf_alloc(size);
2430                         arc_space_consume(size, ARC_SPACE_DATA);
2431                 } else {
2432                         ASSERT(type == ARC_BUFC_DATA);
2433                         buf->b_data = zio_data_buf_alloc(size);
2434                         ARCSTAT_INCR(arcstat_data_size, size);
2435                         atomic_add_64(&arc_size, size);
2436                 }
2437                 goto out;
2438         }
2439 
2440         /*
2441          * If we are prefetching from the mfu ghost list, this buffer
2442          * will end up on the mru list; so steal space from there.
2443          */
2444         if (state == arc_mfu_ghost)
2445                 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
2446         else if (state == arc_mru_ghost)
2447                 state = arc_mru;
2448 
2449         if (state == arc_mru || state == arc_anon) {
2450                 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
2451                 state = (arc_mfu->arcs_lsize[type] >= size &&
2452                     arc_p > mru_used) ? arc_mfu : arc_mru;
2453         } else {
2454                 /* MFU cases */
2455                 uint64_t mfu_space = arc_c - arc_p;
2456                 state =  (arc_mru->arcs_lsize[type] >= size &&
2457                     mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2458         }
2459         if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2460                 if (type == ARC_BUFC_METADATA) {
2461                         buf->b_data = zio_buf_alloc(size);
2462                         arc_space_consume(size, ARC_SPACE_DATA);
2463                 } else {
2464                         ASSERT(type == ARC_BUFC_DATA);
2465                         buf->b_data = zio_data_buf_alloc(size);
2466                         ARCSTAT_INCR(arcstat_data_size, size);
2467                         atomic_add_64(&arc_size, size);
2468                 }
2469                 ARCSTAT_BUMP(arcstat_recycle_miss);
2470         }
2471         ASSERT(buf->b_data != NULL);
2472 out:
2473         /*
2474          * Update the state size.  Note that ghost states have a
2475          * "ghost size" and so don't need to be updated.
2476          */
2477         if (!GHOST_STATE(buf->b_hdr->b_state)) {
2478                 arc_buf_hdr_t *hdr = buf->b_hdr;
2479 
2480                 atomic_add_64(&hdr->b_state->arcs_size, size);
2481                 if (list_link_active(&hdr->b_arc_node)) {
2482                         ASSERT(refcount_is_zero(&hdr->b_refcnt));
2483                         atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2484                 }
2485                 /*
2486                  * If we are growing the cache, and we are adding anonymous
2487                  * data, and we have outgrown arc_p, update arc_p
2488                  */
2489                 if (arc_size < arc_c && hdr->b_state == arc_anon &&
2490                     arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
2491                         arc_p = MIN(arc_c, arc_p + size);
2492         }
2493 }
2494 
2495 /*
2496  * This routine is called whenever a buffer is accessed.
2497  * NOTE: the hash lock is dropped in this function.
2498  */
2499 static void
2500 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2501 {
2502         clock_t now;
2503 
2504         ASSERT(MUTEX_HELD(hash_lock));
2505 
2506         if (buf->b_state == arc_anon) {
2507                 /*
2508                  * This buffer is not in the cache, and does not
2509                  * appear in our "ghost" list.  Add the new buffer
2510                  * to the MRU state.
2511                  */
2512 
2513                 ASSERT(buf->b_arc_access == 0);
2514                 buf->b_arc_access = ddi_get_lbolt();
2515                 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2516                 arc_change_state(arc_mru, buf, hash_lock);
2517 
2518         } else if (buf->b_state == arc_mru) {
2519                 now = ddi_get_lbolt();
2520 
2521                 /*
2522                  * If this buffer is here because of a prefetch, then either:
2523                  * - clear the flag if this is a "referencing" read
2524                  *   (any subsequent access will bump this into the MFU state).
2525                  * or
2526                  * - move the buffer to the head of the list if this is
2527                  *   another prefetch (to make it less likely to be evicted).
2528                  */
2529                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2530                         if (refcount_count(&buf->b_refcnt) == 0) {
2531                                 ASSERT(list_link_active(&buf->b_arc_node));
2532                         } else {
2533                                 buf->b_flags &= ~ARC_PREFETCH;
2534                                 ARCSTAT_BUMP(arcstat_mru_hits);
2535                         }
2536                         buf->b_arc_access = now;
2537                         return;
2538                 }
2539 
2540                 /*
2541                  * This buffer has been "accessed" only once so far,
2542                  * but it is still in the cache. Move it to the MFU
2543                  * state.
2544                  */
2545                 if (now > buf->b_arc_access + ARC_MINTIME) {
2546                         /*
2547                          * More than 125ms have passed since we
2548                          * instantiated this buffer.  Move it to the
2549                          * most frequently used state.
2550                          */
2551                         buf->b_arc_access = now;
2552                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2553                         arc_change_state(arc_mfu, buf, hash_lock);
2554                 }
2555                 ARCSTAT_BUMP(arcstat_mru_hits);
2556         } else if (buf->b_state == arc_mru_ghost) {
2557                 arc_state_t     *new_state;
2558                 /*
2559                  * This buffer has been "accessed" recently, but
2560                  * was evicted from the cache.  Move it to the
2561                  * MFU state.
2562                  */
2563 
2564                 if (buf->b_flags & ARC_PREFETCH) {
2565                         new_state = arc_mru;
2566                         if (refcount_count(&buf->b_refcnt) > 0)
2567                                 buf->b_flags &= ~ARC_PREFETCH;
2568                         DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2569                 } else {
2570                         new_state = arc_mfu;
2571                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2572                 }
2573 
2574                 buf->b_arc_access = ddi_get_lbolt();
2575                 arc_change_state(new_state, buf, hash_lock);
2576 
2577                 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
2578         } else if (buf->b_state == arc_mfu) {
2579                 /*
2580                  * This buffer has been accessed more than once and is
2581                  * still in the cache.  Keep it in the MFU state.
2582                  *
2583                  * NOTE: an add_reference() that occurred when we did
2584                  * the arc_read() will have kicked this off the list.
2585                  * If it was a prefetch, we will explicitly move it to
2586                  * the head of the list now.
2587                  */
2588                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2589                         ASSERT(refcount_count(&buf->b_refcnt) == 0);
2590                         ASSERT(list_link_active(&buf->b_arc_node));
2591                 }
2592                 ARCSTAT_BUMP(arcstat_mfu_hits);
2593                 buf->b_arc_access = ddi_get_lbolt();
2594         } else if (buf->b_state == arc_mfu_ghost) {
2595                 arc_state_t     *new_state = arc_mfu;
2596                 /*
2597                  * This buffer has been accessed more than once but has
2598                  * been evicted from the cache.  Move it back to the
2599                  * MFU state.
2600                  */
2601 
2602                 if (buf->b_flags & ARC_PREFETCH) {
2603                         /*
2604                          * This is a prefetch access...
2605                          * move this block back to the MRU state.
2606                          */
2607                         ASSERT0(refcount_count(&buf->b_refcnt));
2608                         new_state = arc_mru;
2609                 }
2610 
2611                 buf->b_arc_access = ddi_get_lbolt();
2612                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2613                 arc_change_state(new_state, buf, hash_lock);
2614 
2615                 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2616         } else if (buf->b_state == arc_l2c_only) {
2617                 /*
2618                  * This buffer is on the 2nd Level ARC.
2619                  */
2620 
2621                 buf->b_arc_access = ddi_get_lbolt();
2622                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2623                 arc_change_state(arc_mfu, buf, hash_lock);
2624         } else {
2625                 ASSERT(!"invalid arc state");
2626         }
2627 }
2628 
2629 /* a generic arc_done_func_t which you can use */
2630 /* ARGSUSED */
2631 void
2632 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2633 {
2634         if (zio == NULL || zio->io_error == 0)
2635                 bcopy(buf->b_data, arg, buf->b_hdr->b_size);
2636         VERIFY(arc_buf_remove_ref(buf, arg));
2637 }
2638 
2639 /* a generic arc_done_func_t */
2640 void
2641 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2642 {
2643         arc_buf_t **bufp = arg;
2644         if (zio && zio->io_error) {
2645                 VERIFY(arc_buf_remove_ref(buf, arg));
2646                 *bufp = NULL;
2647         } else {
2648                 *bufp = buf;
2649                 ASSERT(buf->b_data);
2650         }
2651 }
2652 
2653 static void
2654 arc_read_done(zio_t *zio)
2655 {
2656         arc_buf_hdr_t   *hdr, *found;
2657         arc_buf_t       *buf;
2658         arc_buf_t       *abuf;  /* buffer we're assigning to callback */
2659         kmutex_t        *hash_lock;
2660         arc_callback_t  *callback_list, *acb;
2661         int             freeable = FALSE;
2662 
2663         buf = zio->io_private;
2664         hdr = buf->b_hdr;
2665 
2666         /*
2667          * The hdr was inserted into hash-table and removed from lists
2668          * prior to starting I/O.  We should find this header, since
2669          * it's in the hash table, and it should be legit since it's
2670          * not possible to evict it during the I/O.  The only possible
2671          * reason for it not to be found is if we were freed during the
2672          * read.
2673          */
2674         found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2675             &hash_lock);
2676 
2677         ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2678             (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2679             (found == hdr && HDR_L2_READING(hdr)));
2680 
2681         hdr->b_flags &= ~ARC_L2_EVICTED;
2682         if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2683                 hdr->b_flags &= ~ARC_L2CACHE;
2684 
2685         /* byteswap if necessary */
2686         callback_list = hdr->b_acb;
2687         ASSERT(callback_list != NULL);
2688         if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2689                 dmu_object_byteswap_t bswap =
2690                     DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2691                 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2692                     byteswap_uint64_array :
2693                     dmu_ot_byteswap[bswap].ob_func;
2694                 func(buf->b_data, hdr->b_size);
2695         }
2696 
2697         arc_cksum_compute(buf, B_FALSE);
2698         arc_buf_watch(buf);
2699 
2700         if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2701                 /*
2702                  * Only call arc_access on anonymous buffers.  This is because
2703                  * if we've issued an I/O for an evicted buffer, we've already
2704                  * called arc_access (to prevent any simultaneous readers from
2705                  * getting confused).
2706                  */
2707                 arc_access(hdr, hash_lock);
2708         }
2709 
2710         /* create copies of the data buffer for the callers */
2711         abuf = buf;
2712         for (acb = callback_list; acb; acb = acb->acb_next) {
2713                 if (acb->acb_done) {
2714                         if (abuf == NULL) {
2715                                 ARCSTAT_BUMP(arcstat_duplicate_reads);
2716                                 abuf = arc_buf_clone(buf);
2717                         }
2718                         acb->acb_buf = abuf;
2719                         abuf = NULL;
2720                 }
2721         }
2722         hdr->b_acb = NULL;
2723         hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2724         ASSERT(!HDR_BUF_AVAILABLE(hdr));
2725         if (abuf == buf) {
2726                 ASSERT(buf->b_efunc == NULL);
2727                 ASSERT(hdr->b_datacnt == 1);
2728                 hdr->b_flags |= ARC_BUF_AVAILABLE;
2729         }
2730 
2731         ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2732 
2733         if (zio->io_error != 0) {
2734                 hdr->b_flags |= ARC_IO_ERROR;
2735                 if (hdr->b_state != arc_anon)
2736                         arc_change_state(arc_anon, hdr, hash_lock);
2737                 if (HDR_IN_HASH_TABLE(hdr))
2738                         buf_hash_remove(hdr);
2739                 freeable = refcount_is_zero(&hdr->b_refcnt);
2740         }
2741 
2742         /*
2743          * Broadcast before we drop the hash_lock to avoid the possibility
2744          * that the hdr (and hence the cv) might be freed before we get to
2745          * the cv_broadcast().
2746          */
2747         cv_broadcast(&hdr->b_cv);
2748 
2749         if (hash_lock) {
2750                 mutex_exit(hash_lock);
2751         } else {
2752                 /*
2753                  * This block was freed while we waited for the read to
2754                  * complete.  It has been removed from the hash table and
2755                  * moved to the anonymous state (so that it won't show up
2756                  * in the cache).
2757                  */
2758                 ASSERT3P(hdr->b_state, ==, arc_anon);
2759                 freeable = refcount_is_zero(&hdr->b_refcnt);
2760         }
2761 
2762         /* execute each callback and free its structure */
2763         while ((acb = callback_list) != NULL) {
2764                 if (acb->acb_done)
2765                         acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2766 
2767                 if (acb->acb_zio_dummy != NULL) {
2768                         acb->acb_zio_dummy->io_error = zio->io_error;
2769                         zio_nowait(acb->acb_zio_dummy);
2770                 }
2771 
2772                 callback_list = acb->acb_next;
2773                 kmem_free(acb, sizeof (arc_callback_t));
2774         }
2775 
2776         if (freeable)
2777                 arc_hdr_destroy(hdr);
2778 }
2779 
2780 /*
2781  * "Read" the block at the specified DVA (in bp) via the
2782  * cache.  If the block is found in the cache, invoke the provided
2783  * callback immediately and return.  Note that the `zio' parameter
2784  * in the callback will be NULL in this case, since no IO was
2785  * required.  If the block is not in the cache pass the read request
2786  * on to the spa with a substitute callback function, so that the
2787  * requested block will be added to the cache.
2788  *
2789  * If a read request arrives for a block that has a read in-progress,
2790  * either wait for the in-progress read to complete (and return the
2791  * results); or, if this is a read with a "done" func, add a record
2792  * to the read to invoke the "done" func when the read completes,
2793  * and return; or just return.
2794  *
2795  * arc_read_done() will invoke all the requested "done" functions
2796  * for readers of this block.
2797  */
2798 int
2799 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
2800     void *private, int priority, int zio_flags, uint32_t *arc_flags,
2801     const zbookmark_t *zb)
2802 {
2803         arc_buf_hdr_t *hdr;
2804         arc_buf_t *buf = NULL;
2805         kmutex_t *hash_lock;
2806         zio_t *rzio;
2807         uint64_t guid = spa_load_guid(spa);
2808 
2809 top:
2810         hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2811             &hash_lock);
2812         if (hdr && hdr->b_datacnt > 0) {
2813 
2814                 *arc_flags |= ARC_CACHED;
2815 
2816                 if (HDR_IO_IN_PROGRESS(hdr)) {
2817 
2818                         if (*arc_flags & ARC_WAIT) {
2819                                 cv_wait(&hdr->b_cv, hash_lock);
2820                                 mutex_exit(hash_lock);
2821                                 goto top;
2822                         }
2823                         ASSERT(*arc_flags & ARC_NOWAIT);
2824 
2825                         if (done) {
2826                                 arc_callback_t  *acb = NULL;
2827 
2828                                 acb = kmem_zalloc(sizeof (arc_callback_t),
2829                                     KM_SLEEP);
2830                                 acb->acb_done = done;
2831                                 acb->acb_private = private;
2832                                 if (pio != NULL)
2833                                         acb->acb_zio_dummy = zio_null(pio,
2834                                             spa, NULL, NULL, NULL, zio_flags);
2835 
2836                                 ASSERT(acb->acb_done != NULL);
2837                                 acb->acb_next = hdr->b_acb;
2838                                 hdr->b_acb = acb;
2839                                 add_reference(hdr, hash_lock, private);
2840                                 mutex_exit(hash_lock);
2841                                 return (0);
2842                         }
2843                         mutex_exit(hash_lock);
2844                         return (0);
2845                 }
2846 
2847                 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2848 
2849                 if (done) {
2850                         add_reference(hdr, hash_lock, private);
2851                         /*
2852                          * If this block is already in use, create a new
2853                          * copy of the data so that we will be guaranteed
2854                          * that arc_release() will always succeed.
2855                          */
2856                         buf = hdr->b_buf;
2857                         ASSERT(buf);
2858                         ASSERT(buf->b_data);
2859                         if (HDR_BUF_AVAILABLE(hdr)) {
2860                                 ASSERT(buf->b_efunc == NULL);
2861                                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2862                         } else {
2863                                 buf = arc_buf_clone(buf);
2864                         }
2865 
2866                 } else if (*arc_flags & ARC_PREFETCH &&
2867                     refcount_count(&hdr->b_refcnt) == 0) {
2868                         hdr->b_flags |= ARC_PREFETCH;
2869                 }
2870                 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
2871                 arc_access(hdr, hash_lock);
2872                 if (*arc_flags & ARC_L2CACHE)
2873                         hdr->b_flags |= ARC_L2CACHE;
2874                 mutex_exit(hash_lock);
2875                 ARCSTAT_BUMP(arcstat_hits);
2876                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2877                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2878                     data, metadata, hits);
2879 
2880                 if (done)
2881                         done(NULL, buf, private);
2882         } else {
2883                 uint64_t size = BP_GET_LSIZE(bp);
2884                 arc_callback_t  *acb;
2885                 vdev_t *vd = NULL;
2886                 uint64_t addr = 0;
2887                 boolean_t devw = B_FALSE;
2888 
2889                 if (hdr == NULL) {
2890                         /* this block is not in the cache */
2891                         arc_buf_hdr_t   *exists;
2892                         arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2893                         buf = arc_buf_alloc(spa, size, private, type);
2894                         hdr = buf->b_hdr;
2895                         hdr->b_dva = *BP_IDENTITY(bp);
2896                         hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2897                         hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2898                         exists = buf_hash_insert(hdr, &hash_lock);
2899                         if (exists) {
2900                                 /* somebody beat us to the hash insert */
2901                                 mutex_exit(hash_lock);
2902                                 buf_discard_identity(hdr);
2903                                 (void) arc_buf_remove_ref(buf, private);
2904                                 goto top; /* restart the IO request */
2905                         }
2906                         /* if this is a prefetch, we don't have a reference */
2907                         if (*arc_flags & ARC_PREFETCH) {
2908                                 (void) remove_reference(hdr, hash_lock,
2909                                     private);
2910                                 hdr->b_flags |= ARC_PREFETCH;
2911                         }
2912                         if (*arc_flags & ARC_L2CACHE)
2913                                 hdr->b_flags |= ARC_L2CACHE;
2914                         if (BP_GET_LEVEL(bp) > 0)
2915                                 hdr->b_flags |= ARC_INDIRECT;
2916                 } else {
2917                         /* this block is in the ghost cache */
2918                         ASSERT(GHOST_STATE(hdr->b_state));
2919                         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2920                         ASSERT0(refcount_count(&hdr->b_refcnt));
2921                         ASSERT(hdr->b_buf == NULL);
2922 
2923                         /* if this is a prefetch, we don't have a reference */
2924                         if (*arc_flags & ARC_PREFETCH)
2925                                 hdr->b_flags |= ARC_PREFETCH;
2926                         else
2927                                 add_reference(hdr, hash_lock, private);
2928                         if (*arc_flags & ARC_L2CACHE)
2929                                 hdr->b_flags |= ARC_L2CACHE;
2930                         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
2931                         buf->b_hdr = hdr;
2932                         buf->b_data = NULL;
2933                         buf->b_efunc = NULL;
2934                         buf->b_private = NULL;
2935                         buf->b_next = NULL;
2936                         hdr->b_buf = buf;
2937                         ASSERT(hdr->b_datacnt == 0);
2938                         hdr->b_datacnt = 1;
2939                         arc_get_data_buf(buf);
2940                         arc_access(hdr, hash_lock);
2941                 }
2942 
2943                 ASSERT(!GHOST_STATE(hdr->b_state));
2944 
2945                 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2946                 acb->acb_done = done;
2947                 acb->acb_private = private;
2948 
2949                 ASSERT(hdr->b_acb == NULL);
2950                 hdr->b_acb = acb;
2951                 hdr->b_flags |= ARC_IO_IN_PROGRESS;
2952 
2953                 if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL &&
2954                     (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
2955                         devw = hdr->b_l2hdr->b_dev->l2ad_writing;
2956                         addr = hdr->b_l2hdr->b_daddr;
2957                         /*
2958                          * Lock out device removal.
2959                          */
2960                         if (vdev_is_dead(vd) ||
2961                             !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
2962                                 vd = NULL;
2963                 }
2964 
2965                 mutex_exit(hash_lock);
2966 
2967                 /*
2968                  * At this point, we have a level 1 cache miss.  Try again in
2969                  * L2ARC if possible.
2970                  */
2971                 ASSERT3U(hdr->b_size, ==, size);
2972                 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
2973                     uint64_t, size, zbookmark_t *, zb);
2974                 ARCSTAT_BUMP(arcstat_misses);
2975                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2976                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2977                     data, metadata, misses);
2978 
2979                 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
2980                         /*
2981                          * Read from the L2ARC if the following are true:
2982                          * 1. The L2ARC vdev was previously cached.
2983                          * 2. This buffer still has L2ARC metadata.
2984                          * 3. This buffer isn't currently writing to the L2ARC.
2985                          * 4. The L2ARC entry wasn't evicted, which may
2986                          *    also have invalidated the vdev.
2987                          * 5. This isn't prefetch and l2arc_noprefetch is set.
2988                          */
2989                         if (hdr->b_l2hdr != NULL &&
2990                             !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
2991                             !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
2992                                 l2arc_read_callback_t *cb;
2993 
2994                                 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
2995                                 ARCSTAT_BUMP(arcstat_l2_hits);
2996 
2997                                 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
2998                                     KM_SLEEP);
2999                                 cb->l2rcb_buf = buf;
3000                                 cb->l2rcb_spa = spa;
3001                                 cb->l2rcb_bp = *bp;
3002                                 cb->l2rcb_zb = *zb;
3003                                 cb->l2rcb_flags = zio_flags;
3004 
3005                                 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
3006                                     addr + size < vd->vdev_psize -
3007                                     VDEV_LABEL_END_SIZE);
3008 
3009                                 /*
3010                                  * l2arc read.  The SCL_L2ARC lock will be
3011                                  * released by l2arc_read_done().
3012                                  */
3013                                 rzio = zio_read_phys(pio, vd, addr, size,
3014                                     buf->b_data, ZIO_CHECKSUM_OFF,
3015                                     l2arc_read_done, cb, priority, zio_flags |
3016                                     ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL |
3017                                     ZIO_FLAG_DONT_PROPAGATE |
3018                                     ZIO_FLAG_DONT_RETRY, B_FALSE);
3019                                 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
3020                                     zio_t *, rzio);
3021                                 ARCSTAT_INCR(arcstat_l2_read_bytes, size);
3022 
3023                                 if (*arc_flags & ARC_NOWAIT) {
3024                                         zio_nowait(rzio);
3025                                         return (0);
3026                                 }
3027 
3028                                 ASSERT(*arc_flags & ARC_WAIT);
3029                                 if (zio_wait(rzio) == 0)
3030                                         return (0);
3031 
3032                                 /* l2arc read error; goto zio_read() */
3033                         } else {
3034                                 DTRACE_PROBE1(l2arc__miss,
3035                                     arc_buf_hdr_t *, hdr);
3036                                 ARCSTAT_BUMP(arcstat_l2_misses);
3037                                 if (HDR_L2_WRITING(hdr))
3038                                         ARCSTAT_BUMP(arcstat_l2_rw_clash);
3039                                 spa_config_exit(spa, SCL_L2ARC, vd);
3040                         }
3041                 } else {
3042                         if (vd != NULL)
3043                                 spa_config_exit(spa, SCL_L2ARC, vd);
3044                         if (l2arc_ndev != 0) {
3045                                 DTRACE_PROBE1(l2arc__miss,
3046                                     arc_buf_hdr_t *, hdr);
3047                                 ARCSTAT_BUMP(arcstat_l2_misses);
3048                         }
3049                 }
3050 
3051                 rzio = zio_read(pio, spa, bp, buf->b_data, size,
3052                     arc_read_done, buf, priority, zio_flags, zb);
3053 
3054                 if (*arc_flags & ARC_WAIT)
3055                         return (zio_wait(rzio));
3056 
3057                 ASSERT(*arc_flags & ARC_NOWAIT);
3058                 zio_nowait(rzio);
3059         }
3060         return (0);
3061 }
3062 
3063 void
3064 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
3065 {
3066         ASSERT(buf->b_hdr != NULL);
3067         ASSERT(buf->b_hdr->b_state != arc_anon);
3068         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
3069         ASSERT(buf->b_efunc == NULL);
3070         ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
3071 
3072         buf->b_efunc = func;
3073         buf->b_private = private;
3074 }
3075 
3076 /*
3077  * This is used by the DMU to let the ARC know that a buffer is
3078  * being evicted, so the ARC should clean up.  If this arc buf
3079  * is not yet in the evicted state, it will be put there.
3080  */
3081 int
3082 arc_buf_evict(arc_buf_t *buf)
3083 {
3084         arc_buf_hdr_t *hdr;
3085         kmutex_t *hash_lock;
3086         arc_buf_t **bufp;
3087 
3088         mutex_enter(&buf->b_evict_lock);
3089         hdr = buf->b_hdr;
3090         if (hdr == NULL) {
3091                 /*
3092                  * We are in arc_do_user_evicts().
3093                  */
3094                 ASSERT(buf->b_data == NULL);
3095                 mutex_exit(&buf->b_evict_lock);
3096                 return (0);
3097         } else if (buf->b_data == NULL) {
3098                 arc_buf_t copy = *buf; /* structure assignment */
3099                 /*
3100                  * We are on the eviction list; process this buffer now
3101                  * but let arc_do_user_evicts() do the reaping.
3102                  */
3103                 buf->b_efunc = NULL;
3104                 mutex_exit(&buf->b_evict_lock);
3105                 VERIFY(copy.b_efunc(&copy) == 0);
3106                 return (1);
3107         }
3108         hash_lock = HDR_LOCK(hdr);
3109         mutex_enter(hash_lock);
3110         hdr = buf->b_hdr;
3111         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3112 
3113         ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
3114         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3115 
3116         /*
3117          * Pull this buffer off of the hdr
3118          */
3119         bufp = &hdr->b_buf;
3120         while (*bufp != buf)
3121                 bufp = &(*bufp)->b_next;
3122         *bufp = buf->b_next;
3123 
3124         ASSERT(buf->b_data != NULL);
3125         arc_buf_destroy(buf, FALSE, FALSE);
3126 
3127         if (hdr->b_datacnt == 0) {
3128                 arc_state_t *old_state = hdr->b_state;
3129                 arc_state_t *evicted_state;
3130 
3131                 ASSERT(hdr->b_buf == NULL);
3132                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
3133 
3134                 evicted_state =
3135                     (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3136 
3137                 mutex_enter(&old_state->arcs_mtx);
3138                 mutex_enter(&evicted_state->arcs_mtx);
3139 
3140                 arc_change_state(evicted_state, hdr, hash_lock);
3141                 ASSERT(HDR_IN_HASH_TABLE(hdr));
3142                 hdr->b_flags |= ARC_IN_HASH_TABLE;
3143                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3144 
3145                 mutex_exit(&evicted_state->arcs_mtx);
3146                 mutex_exit(&old_state->arcs_mtx);
3147         }
3148         mutex_exit(hash_lock);
3149         mutex_exit(&buf->b_evict_lock);
3150 
3151         VERIFY(buf->b_efunc(buf) == 0);
3152         buf->b_efunc = NULL;
3153         buf->b_private = NULL;
3154         buf->b_hdr = NULL;
3155         buf->b_next = NULL;
3156         kmem_cache_free(buf_cache, buf);
3157         return (1);
3158 }
3159 
3160 /*
3161  * Release this buffer from the cache, making it an anonymous buffer.  This
3162  * must be done after a read and prior to modifying the buffer contents.
3163  * If the buffer has more than one reference, we must make
3164  * a new hdr for the buffer.
3165  */
3166 void
3167 arc_release(arc_buf_t *buf, void *tag)
3168 {
3169         arc_buf_hdr_t *hdr;
3170         kmutex_t *hash_lock = NULL;
3171         l2arc_buf_hdr_t *l2hdr;
3172         uint64_t buf_size;
3173 
3174         /*
3175          * It would be nice to assert that if it's DMU metadata (level >
3176          * 0 || it's the dnode file), then it must be syncing context.
3177          * But we don't know that information at this level.
3178          */
3179 
3180         mutex_enter(&buf->b_evict_lock);
3181         hdr = buf->b_hdr;
3182 
3183         /* this buffer is not on any list */
3184         ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3185 
3186         if (hdr->b_state == arc_anon) {
3187                 /* this buffer is already released */
3188                 ASSERT(buf->b_efunc == NULL);
3189         } else {
3190                 hash_lock = HDR_LOCK(hdr);
3191                 mutex_enter(hash_lock);
3192                 hdr = buf->b_hdr;
3193                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3194         }
3195 
3196         l2hdr = hdr->b_l2hdr;
3197         if (l2hdr) {
3198                 mutex_enter(&l2arc_buflist_mtx);
3199                 hdr->b_l2hdr = NULL;
3200         }
3201         buf_size = hdr->b_size;
3202 
3203         /*
3204          * Do we have more than one buf?
3205          */
3206         if (hdr->b_datacnt > 1) {
3207                 arc_buf_hdr_t *nhdr;
3208                 arc_buf_t **bufp;
3209                 uint64_t blksz = hdr->b_size;
3210                 uint64_t spa = hdr->b_spa;
3211                 arc_buf_contents_t type = hdr->b_type;
3212                 uint32_t flags = hdr->b_flags;
3213 
3214                 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3215                 /*
3216                  * Pull the data off of this hdr and attach it to
3217                  * a new anonymous hdr.
3218                  */
3219                 (void) remove_reference(hdr, hash_lock, tag);
3220                 bufp = &hdr->b_buf;
3221                 while (*bufp != buf)
3222                         bufp = &(*bufp)->b_next;
3223                 *bufp = buf->b_next;
3224                 buf->b_next = NULL;
3225 
3226                 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3227                 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3228                 if (refcount_is_zero(&hdr->b_refcnt)) {
3229                         uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3230                         ASSERT3U(*size, >=, hdr->b_size);
3231                         atomic_add_64(size, -hdr->b_size);
3232                 }
3233 
3234                 /*
3235                  * We're releasing a duplicate user data buffer, update
3236                  * our statistics accordingly.
3237                  */
3238                 if (hdr->b_type == ARC_BUFC_DATA) {
3239                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
3240                         ARCSTAT_INCR(arcstat_duplicate_buffers_size,
3241                             -hdr->b_size);
3242                 }
3243                 hdr->b_datacnt -= 1;
3244                 arc_cksum_verify(buf);
3245                 arc_buf_unwatch(buf);
3246 
3247                 mutex_exit(hash_lock);
3248 
3249                 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3250                 nhdr->b_size = blksz;
3251                 nhdr->b_spa = spa;
3252                 nhdr->b_type = type;
3253                 nhdr->b_buf = buf;
3254                 nhdr->b_state = arc_anon;
3255                 nhdr->b_arc_access = 0;
3256                 nhdr->b_flags = flags & ARC_L2_WRITING;
3257                 nhdr->b_l2hdr = NULL;
3258                 nhdr->b_datacnt = 1;
3259                 nhdr->b_freeze_cksum = NULL;
3260                 (void) refcount_add(&nhdr->b_refcnt, tag);
3261                 buf->b_hdr = nhdr;
3262                 mutex_exit(&buf->b_evict_lock);
3263                 atomic_add_64(&arc_anon->arcs_size, blksz);
3264         } else {
3265                 mutex_exit(&buf->b_evict_lock);
3266                 ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3267                 ASSERT(!list_link_active(&hdr->b_arc_node));
3268                 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3269                 if (hdr->b_state != arc_anon)
3270                         arc_change_state(arc_anon, hdr, hash_lock);
3271                 hdr->b_arc_access = 0;
3272                 if (hash_lock)
3273                         mutex_exit(hash_lock);
3274 
3275                 buf_discard_identity(hdr);
3276                 arc_buf_thaw(buf);
3277         }
3278         buf->b_efunc = NULL;
3279         buf->b_private = NULL;
3280 
3281         if (l2hdr) {
3282                 list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3283                 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3284                 ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3285                 mutex_exit(&l2arc_buflist_mtx);
3286         }
3287 }
3288 
3289 int
3290 arc_released(arc_buf_t *buf)
3291 {
3292         int released;
3293 
3294         mutex_enter(&buf->b_evict_lock);
3295         released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
3296         mutex_exit(&buf->b_evict_lock);
3297         return (released);
3298 }
3299 
3300 int
3301 arc_has_callback(arc_buf_t *buf)
3302 {
3303         int callback;
3304 
3305         mutex_enter(&buf->b_evict_lock);
3306         callback = (buf->b_efunc != NULL);
3307         mutex_exit(&buf->b_evict_lock);
3308         return (callback);
3309 }
3310 
3311 #ifdef ZFS_DEBUG
3312 int
3313 arc_referenced(arc_buf_t *buf)
3314 {
3315         int referenced;
3316 
3317         mutex_enter(&buf->b_evict_lock);
3318         referenced = (refcount_count(&buf->b_hdr->b_refcnt));
3319         mutex_exit(&buf->b_evict_lock);
3320         return (referenced);
3321 }
3322 #endif
3323 
3324 static void
3325 arc_write_ready(zio_t *zio)
3326 {
3327         arc_write_callback_t *callback = zio->io_private;
3328         arc_buf_t *buf = callback->awcb_buf;
3329         arc_buf_hdr_t *hdr = buf->b_hdr;
3330 
3331         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3332         callback->awcb_ready(zio, buf, callback->awcb_private);
3333 
3334         /*
3335          * If the IO is already in progress, then this is a re-write
3336          * attempt, so we need to thaw and re-compute the cksum.
3337          * It is the responsibility of the callback to handle the
3338          * accounting for any re-write attempt.
3339          */
3340         if (HDR_IO_IN_PROGRESS(hdr)) {
3341                 mutex_enter(&hdr->b_freeze_lock);
3342                 if (hdr->b_freeze_cksum != NULL) {
3343                         kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3344                         hdr->b_freeze_cksum = NULL;
3345                 }
3346                 mutex_exit(&hdr->b_freeze_lock);
3347         }
3348         arc_cksum_compute(buf, B_FALSE);
3349         hdr->b_flags |= ARC_IO_IN_PROGRESS;
3350 }
3351 
3352 static void
3353 arc_write_done(zio_t *zio)
3354 {
3355         arc_write_callback_t *callback = zio->io_private;
3356         arc_buf_t *buf = callback->awcb_buf;
3357         arc_buf_hdr_t *hdr = buf->b_hdr;
3358 
3359         ASSERT(hdr->b_acb == NULL);
3360 
3361         if (zio->io_error == 0) {
3362                 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3363                 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3364                 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3365         } else {
3366                 ASSERT(BUF_EMPTY(hdr));
3367         }
3368 
3369         /*
3370          * If the block to be written was all-zero, we may have
3371          * compressed it away.  In this case no write was performed
3372          * so there will be no dva/birth/checksum.  The buffer must
3373          * therefore remain anonymous (and uncached).
3374          */
3375         if (!BUF_EMPTY(hdr)) {
3376                 arc_buf_hdr_t *exists;
3377                 kmutex_t *hash_lock;
3378 
3379                 ASSERT(zio->io_error == 0);
3380 
3381                 arc_cksum_verify(buf);
3382 
3383                 exists = buf_hash_insert(hdr, &hash_lock);
3384                 if (exists) {
3385                         /*
3386                          * This can only happen if we overwrite for
3387                          * sync-to-convergence, because we remove
3388                          * buffers from the hash table when we arc_free().
3389                          */
3390                         if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3391                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3392                                         panic("bad overwrite, hdr=%p exists=%p",
3393                                             (void *)hdr, (void *)exists);
3394                                 ASSERT(refcount_is_zero(&exists->b_refcnt));
3395                                 arc_change_state(arc_anon, exists, hash_lock);
3396                                 mutex_exit(hash_lock);
3397                                 arc_hdr_destroy(exists);
3398                                 exists = buf_hash_insert(hdr, &hash_lock);
3399                                 ASSERT3P(exists, ==, NULL);
3400                         } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
3401                                 /* nopwrite */
3402                                 ASSERT(zio->io_prop.zp_nopwrite);
3403                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3404                                         panic("bad nopwrite, hdr=%p exists=%p",
3405                                             (void *)hdr, (void *)exists);
3406                         } else {
3407                                 /* Dedup */
3408                                 ASSERT(hdr->b_datacnt == 1);
3409                                 ASSERT(hdr->b_state == arc_anon);
3410                                 ASSERT(BP_GET_DEDUP(zio->io_bp));
3411                                 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3412                         }
3413                 }
3414                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3415                 /* if it's not anon, we are doing a scrub */
3416                 if (!exists && hdr->b_state == arc_anon)
3417                         arc_access(hdr, hash_lock);
3418                 mutex_exit(hash_lock);
3419         } else {
3420                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3421         }
3422 
3423         ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3424         callback->awcb_done(zio, buf, callback->awcb_private);
3425 
3426         kmem_free(callback, sizeof (arc_write_callback_t));
3427 }
3428 
3429 zio_t *
3430 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3431     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, const zio_prop_t *zp,
3432     arc_done_func_t *ready, arc_done_func_t *done, void *private,
3433     int priority, int zio_flags, const zbookmark_t *zb)
3434 {
3435         arc_buf_hdr_t *hdr = buf->b_hdr;
3436         arc_write_callback_t *callback;
3437         zio_t *zio;
3438 
3439         ASSERT(ready != NULL);
3440         ASSERT(done != NULL);
3441         ASSERT(!HDR_IO_ERROR(hdr));
3442         ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3443         ASSERT(hdr->b_acb == NULL);
3444         if (l2arc)
3445                 hdr->b_flags |= ARC_L2CACHE;
3446         callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3447         callback->awcb_ready = ready;
3448         callback->awcb_done = done;
3449         callback->awcb_private = private;
3450         callback->awcb_buf = buf;
3451 
3452         zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3453             arc_write_ready, arc_write_done, callback, priority, zio_flags, zb);
3454 
3455         return (zio);
3456 }
3457 
3458 static int
3459 arc_memory_throttle(uint64_t reserve, uint64_t inflight_data, uint64_t txg)
3460 {
3461 #ifdef _KERNEL
3462         uint64_t available_memory = ptob(freemem);
3463         static uint64_t page_load = 0;
3464         static uint64_t last_txg = 0;
3465 
3466 #if defined(__i386)
3467         available_memory =
3468             MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3469 #endif
3470         if (available_memory >= zfs_write_limit_max)
3471                 return (0);
3472 
3473         if (txg > last_txg) {
3474                 last_txg = txg;
3475                 page_load = 0;
3476         }
3477         /*
3478          * If we are in pageout, we know that memory is already tight,
3479          * the arc is already going to be evicting, so we just want to
3480          * continue to let page writes occur as quickly as possible.
3481          */
3482         if (curproc == proc_pageout) {
3483                 if (page_load > MAX(ptob(minfree), available_memory) / 4)
3484                         return (SET_ERROR(ERESTART));
3485                 /* Note: reserve is inflated, so we deflate */
3486                 page_load += reserve / 8;
3487                 return (0);
3488         } else if (page_load > 0 && arc_reclaim_needed()) {
3489                 /* memory is low, delay before restarting */
3490                 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3491                 return (SET_ERROR(EAGAIN));
3492         }
3493         page_load = 0;
3494 
3495         if (arc_size > arc_c_min) {
3496                 uint64_t evictable_memory =
3497                     arc_mru->arcs_lsize[ARC_BUFC_DATA] +
3498                     arc_mru->arcs_lsize[ARC_BUFC_METADATA] +
3499                     arc_mfu->arcs_lsize[ARC_BUFC_DATA] +
3500                     arc_mfu->arcs_lsize[ARC_BUFC_METADATA];
3501                 available_memory += MIN(evictable_memory, arc_size - arc_c_min);
3502         }
3503 
3504         if (inflight_data > available_memory / 4) {
3505                 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3506                 return (SET_ERROR(ERESTART));
3507         }
3508 #endif
3509         return (0);
3510 }
3511 
3512 void
3513 arc_tempreserve_clear(uint64_t reserve)
3514 {
3515         atomic_add_64(&arc_tempreserve, -reserve);
3516         ASSERT((int64_t)arc_tempreserve >= 0);
3517 }
3518 
3519 int
3520 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3521 {
3522         int error;
3523         uint64_t anon_size;
3524 
3525 #ifdef ZFS_DEBUG
3526         /*
3527          * Once in a while, fail for no reason.  Everything should cope.
3528          */
3529         if (spa_get_random(10000) == 0) {
3530                 dprintf("forcing random failure\n");
3531                 return (SET_ERROR(ERESTART));
3532         }
3533 #endif
3534         if (reserve > arc_c/4 && !arc_no_grow)
3535                 arc_c = MIN(arc_c_max, reserve * 4);
3536         if (reserve > arc_c)
3537                 return (SET_ERROR(ENOMEM));
3538 
3539         /*
3540          * Don't count loaned bufs as in flight dirty data to prevent long
3541          * network delays from blocking transactions that are ready to be
3542          * assigned to a txg.
3543          */
3544         anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3545 
3546         /*
3547          * Writes will, almost always, require additional memory allocations
3548          * in order to compress/encrypt/etc the data.  We therefor need to
3549          * make sure that there is sufficient available memory for this.
3550          */
3551         if (error = arc_memory_throttle(reserve, anon_size, txg))
3552                 return (error);
3553 
3554         /*
3555          * Throttle writes when the amount of dirty data in the cache
3556          * gets too large.  We try to keep the cache less than half full
3557          * of dirty blocks so that our sync times don't grow too large.
3558          * Note: if two requests come in concurrently, we might let them
3559          * both succeed, when one of them should fail.  Not a huge deal.
3560          */
3561 
3562         if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3563             anon_size > arc_c / 4) {
3564                 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3565                     "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3566                     arc_tempreserve>>10,
3567                     arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3568                     arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3569                     reserve>>10, arc_c>>10);
3570                 return (SET_ERROR(ERESTART));
3571         }
3572         atomic_add_64(&arc_tempreserve, reserve);
3573         return (0);
3574 }
3575 
3576 void
3577 arc_init(void)
3578 {
3579         mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3580         cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3581 
3582         /* Convert seconds to clock ticks */
3583         arc_min_prefetch_lifespan = 1 * hz;
3584 
3585         /* Start out with 1/8 of all memory */
3586         arc_c = physmem * PAGESIZE / 8;
3587 
3588 #ifdef _KERNEL
3589         /*
3590          * On architectures where the physical memory can be larger
3591          * than the addressable space (intel in 32-bit mode), we may
3592          * need to limit the cache to 1/8 of VM size.
3593          */
3594         arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3595 #endif
3596 
3597         /* set min cache to 1/32 of all memory, or 64MB, whichever is more */
3598         arc_c_min = MAX(arc_c / 4, 64<<20);
3599         /* set max to 3/4 of all memory, or all but 1GB, whichever is more */
3600         if (arc_c * 8 >= 1<<30)
3601                 arc_c_max = (arc_c * 8) - (1<<30);
3602         else
3603                 arc_c_max = arc_c_min;
3604         arc_c_max = MAX(arc_c * 6, arc_c_max);
3605 
3606         /*
3607          * Allow the tunables to override our calculations if they are
3608          * reasonable (ie. over 64MB)
3609          */
3610         if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
3611                 arc_c_max = zfs_arc_max;
3612         if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
3613                 arc_c_min = zfs_arc_min;
3614 
3615         arc_c = arc_c_max;
3616         arc_p = (arc_c >> 1);
3617 
3618         /* limit meta-data to 1/4 of the arc capacity */
3619         arc_meta_limit = arc_c_max / 4;
3620 
3621         /* Allow the tunable to override if it is reasonable */
3622         if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
3623                 arc_meta_limit = zfs_arc_meta_limit;
3624 
3625         if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
3626                 arc_c_min = arc_meta_limit / 2;
3627 
3628         if (zfs_arc_grow_retry > 0)
3629                 arc_grow_retry = zfs_arc_grow_retry;
3630 
3631         if (zfs_arc_shrink_shift > 0)
3632                 arc_shrink_shift = zfs_arc_shrink_shift;
3633 
3634         if (zfs_arc_p_min_shift > 0)
3635                 arc_p_min_shift = zfs_arc_p_min_shift;
3636 
3637         /* if kmem_flags are set, lets try to use less memory */
3638         if (kmem_debugging())
3639                 arc_c = arc_c / 2;
3640         if (arc_c < arc_c_min)
3641                 arc_c = arc_c_min;
3642 
3643         arc_anon = &ARC_anon;
3644         arc_mru = &ARC_mru;
3645         arc_mru_ghost = &ARC_mru_ghost;
3646         arc_mfu = &ARC_mfu;
3647         arc_mfu_ghost = &ARC_mfu_ghost;
3648         arc_l2c_only = &ARC_l2c_only;
3649         arc_size = 0;
3650 
3651         mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3652         mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3653         mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3654         mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3655         mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3656         mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3657 
3658         list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
3659             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3660         list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
3661             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3662         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
3663             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3664         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
3665             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3666         list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
3667             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3668         list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
3669             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3670         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
3671             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3672         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
3673             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3674         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3675             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3676         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3677             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3678 
3679         buf_init();
3680 
3681         arc_thread_exit = 0;
3682         arc_eviction_list = NULL;
3683         mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
3684         bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3685 
3686         arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
3687             sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
3688 
3689         if (arc_ksp != NULL) {
3690                 arc_ksp->ks_data = &arc_stats;
3691                 kstat_install(arc_ksp);
3692         }
3693 
3694         (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3695             TS_RUN, minclsyspri);
3696 
3697         arc_dead = FALSE;
3698         arc_warm = B_FALSE;
3699 
3700         if (zfs_write_limit_max == 0)
3701                 zfs_write_limit_max = ptob(physmem) >> zfs_write_limit_shift;
3702         else
3703                 zfs_write_limit_shift = 0;
3704         mutex_init(&zfs_write_limit_lock, NULL, MUTEX_DEFAULT, NULL);
3705 }
3706 
3707 void
3708 arc_fini(void)
3709 {
3710         mutex_enter(&arc_reclaim_thr_lock);
3711         arc_thread_exit = 1;
3712         while (arc_thread_exit != 0)
3713                 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3714         mutex_exit(&arc_reclaim_thr_lock);
3715 
3716         arc_flush(NULL);
3717 
3718         arc_dead = TRUE;
3719 
3720         if (arc_ksp != NULL) {
3721                 kstat_delete(arc_ksp);
3722                 arc_ksp = NULL;
3723         }
3724 
3725         mutex_destroy(&arc_eviction_mtx);
3726         mutex_destroy(&arc_reclaim_thr_lock);
3727         cv_destroy(&arc_reclaim_thr_cv);
3728 
3729         list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
3730         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
3731         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
3732         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
3733         list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
3734         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
3735         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
3736         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3737 
3738         mutex_destroy(&arc_anon->arcs_mtx);
3739         mutex_destroy(&arc_mru->arcs_mtx);
3740         mutex_destroy(&arc_mru_ghost->arcs_mtx);
3741         mutex_destroy(&arc_mfu->arcs_mtx);
3742         mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3743         mutex_destroy(&arc_l2c_only->arcs_mtx);
3744 
3745         mutex_destroy(&zfs_write_limit_lock);
3746 
3747         buf_fini();
3748 
3749         ASSERT(arc_loaned_bytes == 0);
3750 }
3751 
3752 /*
3753  * Level 2 ARC
3754  *
3755  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3756  * It uses dedicated storage devices to hold cached data, which are populated
3757  * using large infrequent writes.  The main role of this cache is to boost
3758  * the performance of random read workloads.  The intended L2ARC devices
3759  * include short-stroked disks, solid state disks, and other media with
3760  * substantially faster read latency than disk.
3761  *
3762  *                 +-----------------------+
3763  *                 |         ARC           |
3764  *                 +-----------------------+
3765  *                    |         ^     ^
3766  *                    |         |     |
3767  *      l2arc_feed_thread()    arc_read()
3768  *                    |         |     |
3769  *                    |  l2arc read   |
3770  *                    V         |     |
3771  *               +---------------+    |
3772  *               |     L2ARC     |    |
3773  *               +---------------+    |
3774  *                   |    ^           |
3775  *          l2arc_write() |           |
3776  *                   |    |           |
3777  *                   V    |           |
3778  *                 +-------+      +-------+
3779  *                 | vdev  |      | vdev  |
3780  *                 | cache |      | cache |
3781  *                 +-------+      +-------+
3782  *                 +=========+     .-----.
3783  *                 :  L2ARC  :    |-_____-|
3784  *                 : devices :    | Disks |
3785  *                 +=========+    `-_____-'
3786  *
3787  * Read requests are satisfied from the following sources, in order:
3788  *
3789  *      1) ARC
3790  *      2) vdev cache of L2ARC devices
3791  *      3) L2ARC devices
3792  *      4) vdev cache of disks
3793  *      5) disks
3794  *
3795  * Some L2ARC device types exhibit extremely slow write performance.
3796  * To accommodate for this there are some significant differences between
3797  * the L2ARC and traditional cache design:
3798  *
3799  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3800  * the ARC behave as usual, freeing buffers and placing headers on ghost
3801  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3802  * this would add inflated write latencies for all ARC memory pressure.
3803  *
3804  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3805  * It does this by periodically scanning buffers from the eviction-end of
3806  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3807  * not already there.  It scans until a headroom of buffers is satisfied,
3808  * which itself is a buffer for ARC eviction.  The thread that does this is
3809  * l2arc_feed_thread(), illustrated below; example sizes are included to
3810  * provide a better sense of ratio than this diagram:
3811  *
3812  *             head -->                        tail
3813  *              +---------------------+----------+
3814  *      ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3815  *              +---------------------+----------+   |   o L2ARC eligible
3816  *      ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3817  *              +---------------------+----------+   |
3818  *                   15.9 Gbytes      ^ 32 Mbytes    |
3819  *                                 headroom          |
3820  *                                            l2arc_feed_thread()
3821  *                                                   |
3822  *                       l2arc write hand <--[oooo]--'
3823  *                               |           8 Mbyte
3824  *                               |          write max
3825  *                               V
3826  *                +==============================+
3827  *      L2ARC dev |####|#|###|###|    |####| ... |
3828  *                +==============================+
3829  *                           32 Gbytes
3830  *
3831  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3832  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3833  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3834  * safe to say that this is an uncommon case, since buffers at the end of
3835  * the ARC lists have moved there due to inactivity.
3836  *
3837  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3838  * then the L2ARC simply misses copying some buffers.  This serves as a
3839  * pressure valve to prevent heavy read workloads from both stalling the ARC
3840  * with waits and clogging the L2ARC with writes.  This also helps prevent
3841  * the potential for the L2ARC to churn if it attempts to cache content too
3842  * quickly, such as during backups of the entire pool.
3843  *
3844  * 5. After system boot and before the ARC has filled main memory, there are
3845  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
3846  * lists can remain mostly static.  Instead of searching from tail of these
3847  * lists as pictured, the l2arc_feed_thread() will search from the list heads
3848  * for eligible buffers, greatly increasing its chance of finding them.
3849  *
3850  * The L2ARC device write speed is also boosted during this time so that
3851  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
3852  * there are no L2ARC reads, and no fear of degrading read performance
3853  * through increased writes.
3854  *
3855  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
3856  * the vdev queue can aggregate them into larger and fewer writes.  Each
3857  * device is written to in a rotor fashion, sweeping writes through
3858  * available space then repeating.
3859  *
3860  * 7. The L2ARC does not store dirty content.  It never needs to flush
3861  * write buffers back to disk based storage.
3862  *
3863  * 8. If an ARC buffer is written (and dirtied) which also exists in the
3864  * L2ARC, the now stale L2ARC buffer is immediately dropped.
3865  *
3866  * The performance of the L2ARC can be tweaked by a number of tunables, which
3867  * may be necessary for different workloads:
3868  *
3869  *      l2arc_write_max         max write bytes per interval
3870  *      l2arc_write_boost       extra write bytes during device warmup
3871  *      l2arc_noprefetch        skip caching prefetched buffers
3872  *      l2arc_headroom          number of max device writes to precache
3873  *      l2arc_feed_secs         seconds between L2ARC writing
3874  *
3875  * Tunables may be removed or added as future performance improvements are
3876  * integrated, and also may become zpool properties.
3877  *
3878  * There are three key functions that control how the L2ARC warms up:
3879  *
3880  *      l2arc_write_eligible()  check if a buffer is eligible to cache
3881  *      l2arc_write_size()      calculate how much to write
3882  *      l2arc_write_interval()  calculate sleep delay between writes
3883  *
3884  * These three functions determine what to write, how much, and how quickly
3885  * to send writes.
3886  */
3887 
3888 static boolean_t
3889 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
3890 {
3891         /*
3892          * A buffer is *not* eligible for the L2ARC if it:
3893          * 1. belongs to a different spa.
3894          * 2. is already cached on the L2ARC.
3895          * 3. has an I/O in progress (it may be an incomplete read).
3896          * 4. is flagged not eligible (zfs property).
3897          */
3898         if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
3899             HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
3900                 return (B_FALSE);
3901 
3902         return (B_TRUE);
3903 }
3904 
3905 static uint64_t
3906 l2arc_write_size(l2arc_dev_t *dev)
3907 {
3908         uint64_t size;
3909 
3910         size = dev->l2ad_write;
3911 
3912         if (arc_warm == B_FALSE)
3913                 size += dev->l2ad_boost;
3914 
3915         return (size);
3916 
3917 }
3918 
3919 static clock_t
3920 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
3921 {
3922         clock_t interval, next, now;
3923 
3924         /*
3925          * If the ARC lists are busy, increase our write rate; if the
3926          * lists are stale, idle back.  This is achieved by checking
3927          * how much we previously wrote - if it was more than half of
3928          * what we wanted, schedule the next write much sooner.
3929          */
3930         if (l2arc_feed_again && wrote > (wanted / 2))
3931                 interval = (hz * l2arc_feed_min_ms) / 1000;
3932         else
3933                 interval = hz * l2arc_feed_secs;
3934 
3935         now = ddi_get_lbolt();
3936         next = MAX(now, MIN(now + interval, began + interval));
3937 
3938         return (next);
3939 }
3940 
3941 static void
3942 l2arc_hdr_stat_add(void)
3943 {
3944         ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
3945         ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
3946 }
3947 
3948 static void
3949 l2arc_hdr_stat_remove(void)
3950 {
3951         ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
3952         ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
3953 }
3954 
3955 /*
3956  * Cycle through L2ARC devices.  This is how L2ARC load balances.
3957  * If a device is returned, this also returns holding the spa config lock.
3958  */
3959 static l2arc_dev_t *
3960 l2arc_dev_get_next(void)
3961 {
3962         l2arc_dev_t *first, *next = NULL;
3963 
3964         /*
3965          * Lock out the removal of spas (spa_namespace_lock), then removal
3966          * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
3967          * both locks will be dropped and a spa config lock held instead.
3968          */
3969         mutex_enter(&spa_namespace_lock);
3970         mutex_enter(&l2arc_dev_mtx);
3971 
3972         /* if there are no vdevs, there is nothing to do */
3973         if (l2arc_ndev == 0)
3974                 goto out;
3975 
3976         first = NULL;
3977         next = l2arc_dev_last;
3978         do {
3979                 /* loop around the list looking for a non-faulted vdev */
3980                 if (next == NULL) {
3981                         next = list_head(l2arc_dev_list);
3982                 } else {
3983                         next = list_next(l2arc_dev_list, next);
3984                         if (next == NULL)
3985                                 next = list_head(l2arc_dev_list);
3986                 }
3987 
3988                 /* if we have come back to the start, bail out */
3989                 if (first == NULL)
3990                         first = next;
3991                 else if (next == first)
3992                         break;
3993 
3994         } while (vdev_is_dead(next->l2ad_vdev));
3995 
3996         /* if we were unable to find any usable vdevs, return NULL */
3997         if (vdev_is_dead(next->l2ad_vdev))
3998                 next = NULL;
3999 
4000         l2arc_dev_last = next;
4001 
4002 out:
4003         mutex_exit(&l2arc_dev_mtx);
4004 
4005         /*
4006          * Grab the config lock to prevent the 'next' device from being
4007          * removed while we are writing to it.
4008          */
4009         if (next != NULL)
4010                 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
4011         mutex_exit(&spa_namespace_lock);
4012 
4013         return (next);
4014 }
4015 
4016 /*
4017  * Free buffers that were tagged for destruction.
4018  */
4019 static void
4020 l2arc_do_free_on_write()
4021 {
4022         list_t *buflist;
4023         l2arc_data_free_t *df, *df_prev;
4024 
4025         mutex_enter(&l2arc_free_on_write_mtx);
4026         buflist = l2arc_free_on_write;
4027 
4028         for (df = list_tail(buflist); df; df = df_prev) {
4029                 df_prev = list_prev(buflist, df);
4030                 ASSERT(df->l2df_data != NULL);
4031                 ASSERT(df->l2df_func != NULL);
4032                 df->l2df_func(df->l2df_data, df->l2df_size);
4033                 list_remove(buflist, df);
4034                 kmem_free(df, sizeof (l2arc_data_free_t));
4035         }
4036 
4037         mutex_exit(&l2arc_free_on_write_mtx);
4038 }
4039 
4040 /*
4041  * A write to a cache device has completed.  Update all headers to allow
4042  * reads from these buffers to begin.
4043  */
4044 static void
4045 l2arc_write_done(zio_t *zio)
4046 {
4047         l2arc_write_callback_t *cb;
4048         l2arc_dev_t *dev;
4049         list_t *buflist;
4050         arc_buf_hdr_t *head, *ab, *ab_prev;
4051         l2arc_buf_hdr_t *abl2;
4052         kmutex_t *hash_lock;
4053 
4054         cb = zio->io_private;
4055         ASSERT(cb != NULL);
4056         dev = cb->l2wcb_dev;
4057         ASSERT(dev != NULL);
4058         head = cb->l2wcb_head;
4059         ASSERT(head != NULL);
4060         buflist = dev->l2ad_buflist;
4061         ASSERT(buflist != NULL);
4062         DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
4063             l2arc_write_callback_t *, cb);
4064 
4065         if (zio->io_error != 0)
4066                 ARCSTAT_BUMP(arcstat_l2_writes_error);
4067 
4068         mutex_enter(&l2arc_buflist_mtx);
4069 
4070         /*
4071          * All writes completed, or an error was hit.
4072          */
4073         for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
4074                 ab_prev = list_prev(buflist, ab);
4075 
4076                 hash_lock = HDR_LOCK(ab);
4077                 if (!mutex_tryenter(hash_lock)) {
4078                         /*
4079                          * This buffer misses out.  It may be in a stage
4080                          * of eviction.  Its ARC_L2_WRITING flag will be
4081                          * left set, denying reads to this buffer.
4082                          */
4083                         ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
4084                         continue;
4085                 }
4086 
4087                 if (zio->io_error != 0) {
4088                         /*
4089                          * Error - drop L2ARC entry.
4090                          */
4091                         list_remove(buflist, ab);
4092                         abl2 = ab->b_l2hdr;
4093                         ab->b_l2hdr = NULL;
4094                         kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4095                         ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4096                 }
4097 
4098                 /*
4099                  * Allow ARC to begin reads to this L2ARC entry.
4100                  */
4101                 ab->b_flags &= ~ARC_L2_WRITING;
4102 
4103                 mutex_exit(hash_lock);
4104         }
4105 
4106         atomic_inc_64(&l2arc_writes_done);
4107         list_remove(buflist, head);
4108         kmem_cache_free(hdr_cache, head);
4109         mutex_exit(&l2arc_buflist_mtx);
4110 
4111         l2arc_do_free_on_write();
4112 
4113         kmem_free(cb, sizeof (l2arc_write_callback_t));
4114 }
4115 
4116 /*
4117  * A read to a cache device completed.  Validate buffer contents before
4118  * handing over to the regular ARC routines.
4119  */
4120 static void
4121 l2arc_read_done(zio_t *zio)
4122 {
4123         l2arc_read_callback_t *cb;
4124         arc_buf_hdr_t *hdr;
4125         arc_buf_t *buf;
4126         kmutex_t *hash_lock;
4127         int equal;
4128 
4129         ASSERT(zio->io_vd != NULL);
4130         ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
4131 
4132         spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
4133 
4134         cb = zio->io_private;
4135         ASSERT(cb != NULL);
4136         buf = cb->l2rcb_buf;
4137         ASSERT(buf != NULL);
4138 
4139         hash_lock = HDR_LOCK(buf->b_hdr);
4140         mutex_enter(hash_lock);
4141         hdr = buf->b_hdr;
4142         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4143 
4144         /*
4145          * Check this survived the L2ARC journey.
4146          */
4147         equal = arc_cksum_equal(buf);
4148         if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
4149                 mutex_exit(hash_lock);
4150                 zio->io_private = buf;
4151                 zio->io_bp_copy = cb->l2rcb_bp;   /* XXX fix in L2ARC 2.0 */
4152                 zio->io_bp = &zio->io_bp_copy;        /* XXX fix in L2ARC 2.0 */
4153                 arc_read_done(zio);
4154         } else {
4155                 mutex_exit(hash_lock);
4156                 /*
4157                  * Buffer didn't survive caching.  Increment stats and
4158                  * reissue to the original storage device.
4159                  */
4160                 if (zio->io_error != 0) {
4161                         ARCSTAT_BUMP(arcstat_l2_io_error);
4162                 } else {
4163                         zio->io_error = SET_ERROR(EIO);
4164                 }
4165                 if (!equal)
4166                         ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4167 
4168                 /*
4169                  * If there's no waiter, issue an async i/o to the primary
4170                  * storage now.  If there *is* a waiter, the caller must
4171                  * issue the i/o in a context where it's OK to block.
4172                  */
4173                 if (zio->io_waiter == NULL) {
4174                         zio_t *pio = zio_unique_parent(zio);
4175 
4176                         ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4177 
4178                         zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4179                             buf->b_data, zio->io_size, arc_read_done, buf,
4180                             zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4181                 }
4182         }
4183 
4184         kmem_free(cb, sizeof (l2arc_read_callback_t));
4185 }
4186 
4187 /*
4188  * This is the list priority from which the L2ARC will search for pages to
4189  * cache.  This is used within loops (0..3) to cycle through lists in the
4190  * desired order.  This order can have a significant effect on cache
4191  * performance.
4192  *
4193  * Currently the metadata lists are hit first, MFU then MRU, followed by
4194  * the data lists.  This function returns a locked list, and also returns
4195  * the lock pointer.
4196  */
4197 static list_t *
4198 l2arc_list_locked(int list_num, kmutex_t **lock)
4199 {
4200         list_t *list = NULL;
4201 
4202         ASSERT(list_num >= 0 && list_num <= 3);
4203 
4204         switch (list_num) {
4205         case 0:
4206                 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4207                 *lock = &arc_mfu->arcs_mtx;
4208                 break;
4209         case 1:
4210                 list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4211                 *lock = &arc_mru->arcs_mtx;
4212                 break;
4213         case 2:
4214                 list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4215                 *lock = &arc_mfu->arcs_mtx;
4216                 break;
4217         case 3:
4218                 list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4219                 *lock = &arc_mru->arcs_mtx;
4220                 break;
4221         }
4222 
4223         ASSERT(!(MUTEX_HELD(*lock)));
4224         mutex_enter(*lock);
4225         return (list);
4226 }
4227 
4228 /*
4229  * Evict buffers from the device write hand to the distance specified in
4230  * bytes.  This distance may span populated buffers, it may span nothing.
4231  * This is clearing a region on the L2ARC device ready for writing.
4232  * If the 'all' boolean is set, every buffer is evicted.
4233  */
4234 static void
4235 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4236 {
4237         list_t *buflist;
4238         l2arc_buf_hdr_t *abl2;
4239         arc_buf_hdr_t *ab, *ab_prev;
4240         kmutex_t *hash_lock;
4241         uint64_t taddr;
4242 
4243         buflist = dev->l2ad_buflist;
4244 
4245         if (buflist == NULL)
4246                 return;
4247 
4248         if (!all && dev->l2ad_first) {
4249                 /*
4250                  * This is the first sweep through the device.  There is
4251                  * nothing to evict.
4252                  */
4253                 return;
4254         }
4255 
4256         if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4257                 /*
4258                  * When nearing the end of the device, evict to the end
4259                  * before the device write hand jumps to the start.
4260                  */
4261                 taddr = dev->l2ad_end;
4262         } else {
4263                 taddr = dev->l2ad_hand + distance;
4264         }
4265         DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4266             uint64_t, taddr, boolean_t, all);
4267 
4268 top:
4269         mutex_enter(&l2arc_buflist_mtx);
4270         for (ab = list_tail(buflist); ab; ab = ab_prev) {
4271                 ab_prev = list_prev(buflist, ab);
4272 
4273                 hash_lock = HDR_LOCK(ab);
4274                 if (!mutex_tryenter(hash_lock)) {
4275                         /*
4276                          * Missed the hash lock.  Retry.
4277                          */
4278                         ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4279                         mutex_exit(&l2arc_buflist_mtx);
4280                         mutex_enter(hash_lock);
4281                         mutex_exit(hash_lock);
4282                         goto top;
4283                 }
4284 
4285                 if (HDR_L2_WRITE_HEAD(ab)) {
4286                         /*
4287                          * We hit a write head node.  Leave it for
4288                          * l2arc_write_done().
4289                          */
4290                         list_remove(buflist, ab);
4291                         mutex_exit(hash_lock);
4292                         continue;
4293                 }
4294 
4295                 if (!all && ab->b_l2hdr != NULL &&
4296                     (ab->b_l2hdr->b_daddr > taddr ||
4297                     ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4298                         /*
4299                          * We've evicted to the target address,
4300                          * or the end of the device.
4301                          */
4302                         mutex_exit(hash_lock);
4303                         break;
4304                 }
4305 
4306                 if (HDR_FREE_IN_PROGRESS(ab)) {
4307                         /*
4308                          * Already on the path to destruction.
4309                          */
4310                         mutex_exit(hash_lock);
4311                         continue;
4312                 }
4313 
4314                 if (ab->b_state == arc_l2c_only) {
4315                         ASSERT(!HDR_L2_READING(ab));
4316                         /*
4317                          * This doesn't exist in the ARC.  Destroy.
4318                          * arc_hdr_destroy() will call list_remove()
4319                          * and decrement arcstat_l2_size.
4320                          */
4321                         arc_change_state(arc_anon, ab, hash_lock);
4322                         arc_hdr_destroy(ab);
4323                 } else {
4324                         /*
4325                          * Invalidate issued or about to be issued
4326                          * reads, since we may be about to write
4327                          * over this location.
4328                          */
4329                         if (HDR_L2_READING(ab)) {
4330                                 ARCSTAT_BUMP(arcstat_l2_evict_reading);
4331                                 ab->b_flags |= ARC_L2_EVICTED;
4332                         }
4333 
4334                         /*
4335                          * Tell ARC this no longer exists in L2ARC.
4336                          */
4337                         if (ab->b_l2hdr != NULL) {
4338                                 abl2 = ab->b_l2hdr;
4339                                 ab->b_l2hdr = NULL;
4340                                 kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4341                                 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4342                         }
4343                         list_remove(buflist, ab);
4344 
4345                         /*
4346                          * This may have been leftover after a
4347                          * failed write.
4348                          */
4349                         ab->b_flags &= ~ARC_L2_WRITING;
4350                 }
4351                 mutex_exit(hash_lock);
4352         }
4353         mutex_exit(&l2arc_buflist_mtx);
4354 
4355         vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4356         dev->l2ad_evict = taddr;
4357 }
4358 
4359 /*
4360  * Find and write ARC buffers to the L2ARC device.
4361  *
4362  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4363  * for reading until they have completed writing.
4364  */
4365 static uint64_t
4366 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
4367 {
4368         arc_buf_hdr_t *ab, *ab_prev, *head;
4369         l2arc_buf_hdr_t *hdrl2;
4370         list_t *list;
4371         uint64_t passed_sz, write_sz, buf_sz, headroom;
4372         void *buf_data;
4373         kmutex_t *hash_lock, *list_lock;
4374         boolean_t have_lock, full;
4375         l2arc_write_callback_t *cb;
4376         zio_t *pio, *wzio;
4377         uint64_t guid = spa_load_guid(spa);
4378 
4379         ASSERT(dev->l2ad_vdev != NULL);
4380 
4381         pio = NULL;
4382         write_sz = 0;
4383         full = B_FALSE;
4384         head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4385         head->b_flags |= ARC_L2_WRITE_HEAD;
4386 
4387         /*
4388          * Copy buffers for L2ARC writing.
4389          */
4390         mutex_enter(&l2arc_buflist_mtx);
4391         for (int try = 0; try <= 3; try++) {
4392                 list = l2arc_list_locked(try, &list_lock);
4393                 passed_sz = 0;
4394 
4395                 /*
4396                  * L2ARC fast warmup.
4397                  *
4398                  * Until the ARC is warm and starts to evict, read from the
4399                  * head of the ARC lists rather than the tail.
4400                  */
4401                 headroom = target_sz * l2arc_headroom;
4402                 if (arc_warm == B_FALSE)
4403                         ab = list_head(list);
4404                 else
4405                         ab = list_tail(list);
4406 
4407                 for (; ab; ab = ab_prev) {
4408                         if (arc_warm == B_FALSE)
4409                                 ab_prev = list_next(list, ab);
4410                         else
4411                                 ab_prev = list_prev(list, ab);
4412 
4413                         hash_lock = HDR_LOCK(ab);
4414                         have_lock = MUTEX_HELD(hash_lock);
4415                         if (!have_lock && !mutex_tryenter(hash_lock)) {
4416                                 /*
4417                                  * Skip this buffer rather than waiting.
4418                                  */
4419                                 continue;
4420                         }
4421 
4422                         passed_sz += ab->b_size;
4423                         if (passed_sz > headroom) {
4424                                 /*
4425                                  * Searched too far.
4426                                  */
4427                                 mutex_exit(hash_lock);
4428                                 break;
4429                         }
4430 
4431                         if (!l2arc_write_eligible(guid, ab)) {
4432                                 mutex_exit(hash_lock);
4433                                 continue;
4434                         }
4435 
4436                         if ((write_sz + ab->b_size) > target_sz) {
4437                                 full = B_TRUE;
4438                                 mutex_exit(hash_lock);
4439                                 break;
4440                         }
4441 
4442                         if (pio == NULL) {
4443                                 /*
4444                                  * Insert a dummy header on the buflist so
4445                                  * l2arc_write_done() can find where the
4446                                  * write buffers begin without searching.
4447                                  */
4448                                 list_insert_head(dev->l2ad_buflist, head);
4449 
4450                                 cb = kmem_alloc(
4451                                     sizeof (l2arc_write_callback_t), KM_SLEEP);
4452                                 cb->l2wcb_dev = dev;
4453                                 cb->l2wcb_head = head;
4454                                 pio = zio_root(spa, l2arc_write_done, cb,
4455                                     ZIO_FLAG_CANFAIL);
4456                         }
4457 
4458                         /*
4459                          * Create and add a new L2ARC header.
4460                          */
4461                         hdrl2 = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4462                         hdrl2->b_dev = dev;
4463                         hdrl2->b_daddr = dev->l2ad_hand;
4464 
4465                         ab->b_flags |= ARC_L2_WRITING;
4466                         ab->b_l2hdr = hdrl2;
4467                         list_insert_head(dev->l2ad_buflist, ab);
4468                         buf_data = ab->b_buf->b_data;
4469                         buf_sz = ab->b_size;
4470 
4471                         /*
4472                          * Compute and store the buffer cksum before
4473                          * writing.  On debug the cksum is verified first.
4474                          */
4475                         arc_cksum_verify(ab->b_buf);
4476                         arc_cksum_compute(ab->b_buf, B_TRUE);
4477 
4478                         mutex_exit(hash_lock);
4479 
4480                         wzio = zio_write_phys(pio, dev->l2ad_vdev,
4481                             dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4482                             NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4483                             ZIO_FLAG_CANFAIL, B_FALSE);
4484 
4485                         DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4486                             zio_t *, wzio);
4487                         (void) zio_nowait(wzio);
4488 
4489                         /*
4490                          * Keep the clock hand suitably device-aligned.
4491                          */
4492                         buf_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4493 
4494                         write_sz += buf_sz;
4495                         dev->l2ad_hand += buf_sz;
4496                 }
4497 
4498                 mutex_exit(list_lock);
4499 
4500                 if (full == B_TRUE)
4501                         break;
4502         }
4503         mutex_exit(&l2arc_buflist_mtx);
4504 
4505         if (pio == NULL) {
4506                 ASSERT0(write_sz);
4507                 kmem_cache_free(hdr_cache, head);
4508                 return (0);
4509         }
4510 
4511         ASSERT3U(write_sz, <=, target_sz);
4512         ARCSTAT_BUMP(arcstat_l2_writes_sent);
4513         ARCSTAT_INCR(arcstat_l2_write_bytes, write_sz);
4514         ARCSTAT_INCR(arcstat_l2_size, write_sz);
4515         vdev_space_update(dev->l2ad_vdev, write_sz, 0, 0);
4516 
4517         /*
4518          * Bump device hand to the device start if it is approaching the end.
4519          * l2arc_evict() will already have evicted ahead for this case.
4520          */
4521         if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4522                 vdev_space_update(dev->l2ad_vdev,
4523                     dev->l2ad_end - dev->l2ad_hand, 0, 0);
4524                 dev->l2ad_hand = dev->l2ad_start;
4525                 dev->l2ad_evict = dev->l2ad_start;
4526                 dev->l2ad_first = B_FALSE;
4527         }
4528 
4529         dev->l2ad_writing = B_TRUE;
4530         (void) zio_wait(pio);
4531         dev->l2ad_writing = B_FALSE;
4532 
4533         return (write_sz);
4534 }
4535 
4536 /*
4537  * This thread feeds the L2ARC at regular intervals.  This is the beating
4538  * heart of the L2ARC.
4539  */
4540 static void
4541 l2arc_feed_thread(void)
4542 {
4543         callb_cpr_t cpr;
4544         l2arc_dev_t *dev;
4545         spa_t *spa;
4546         uint64_t size, wrote;
4547         clock_t begin, next = ddi_get_lbolt();
4548 
4549         CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4550 
4551         mutex_enter(&l2arc_feed_thr_lock);
4552 
4553         while (l2arc_thread_exit == 0) {
4554                 CALLB_CPR_SAFE_BEGIN(&cpr);
4555                 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
4556                     next);
4557                 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4558                 next = ddi_get_lbolt() + hz;
4559 
4560                 /*
4561                  * Quick check for L2ARC devices.
4562                  */
4563                 mutex_enter(&l2arc_dev_mtx);
4564                 if (l2arc_ndev == 0) {
4565                         mutex_exit(&l2arc_dev_mtx);
4566                         continue;
4567                 }
4568                 mutex_exit(&l2arc_dev_mtx);
4569                 begin = ddi_get_lbolt();
4570 
4571                 /*
4572                  * This selects the next l2arc device to write to, and in
4573                  * doing so the next spa to feed from: dev->l2ad_spa.   This
4574                  * will return NULL if there are now no l2arc devices or if
4575                  * they are all faulted.
4576                  *
4577                  * If a device is returned, its spa's config lock is also
4578                  * held to prevent device removal.  l2arc_dev_get_next()
4579                  * will grab and release l2arc_dev_mtx.
4580                  */
4581                 if ((dev = l2arc_dev_get_next()) == NULL)
4582                         continue;
4583 
4584                 spa = dev->l2ad_spa;
4585                 ASSERT(spa != NULL);
4586 
4587                 /*
4588                  * If the pool is read-only then force the feed thread to
4589                  * sleep a little longer.
4590                  */
4591                 if (!spa_writeable(spa)) {
4592                         next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
4593                         spa_config_exit(spa, SCL_L2ARC, dev);
4594                         continue;
4595                 }
4596 
4597                 /*
4598                  * Avoid contributing to memory pressure.
4599                  */
4600                 if (arc_reclaim_needed()) {
4601                         ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
4602                         spa_config_exit(spa, SCL_L2ARC, dev);
4603                         continue;
4604                 }
4605 
4606                 ARCSTAT_BUMP(arcstat_l2_feeds);
4607 
4608                 size = l2arc_write_size(dev);
4609 
4610                 /*
4611                  * Evict L2ARC buffers that will be overwritten.
4612                  */
4613                 l2arc_evict(dev, size, B_FALSE);
4614 
4615                 /*
4616                  * Write ARC buffers.
4617                  */
4618                 wrote = l2arc_write_buffers(spa, dev, size);
4619 
4620                 /*
4621                  * Calculate interval between writes.
4622                  */
4623                 next = l2arc_write_interval(begin, size, wrote);
4624                 spa_config_exit(spa, SCL_L2ARC, dev);
4625         }
4626 
4627         l2arc_thread_exit = 0;
4628         cv_broadcast(&l2arc_feed_thr_cv);
4629         CALLB_CPR_EXIT(&cpr);               /* drops l2arc_feed_thr_lock */
4630         thread_exit();
4631 }
4632 
4633 boolean_t
4634 l2arc_vdev_present(vdev_t *vd)
4635 {
4636         l2arc_dev_t *dev;
4637 
4638         mutex_enter(&l2arc_dev_mtx);
4639         for (dev = list_head(l2arc_dev_list); dev != NULL;
4640             dev = list_next(l2arc_dev_list, dev)) {
4641                 if (dev->l2ad_vdev == vd)
4642                         break;
4643         }
4644         mutex_exit(&l2arc_dev_mtx);
4645 
4646         return (dev != NULL);
4647 }
4648 
4649 /*
4650  * Add a vdev for use by the L2ARC.  By this point the spa has already
4651  * validated the vdev and opened it.
4652  */
4653 void
4654 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
4655 {
4656         l2arc_dev_t *adddev;
4657 
4658         ASSERT(!l2arc_vdev_present(vd));
4659 
4660         /*
4661          * Create a new l2arc device entry.
4662          */
4663         adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
4664         adddev->l2ad_spa = spa;
4665         adddev->l2ad_vdev = vd;
4666         adddev->l2ad_write = l2arc_write_max;
4667         adddev->l2ad_boost = l2arc_write_boost;
4668         adddev->l2ad_start = VDEV_LABEL_START_SIZE;
4669         adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
4670         adddev->l2ad_hand = adddev->l2ad_start;
4671         adddev->l2ad_evict = adddev->l2ad_start;
4672         adddev->l2ad_first = B_TRUE;
4673         adddev->l2ad_writing = B_FALSE;
4674         ASSERT3U(adddev->l2ad_write, >, 0);
4675 
4676         /*
4677          * This is a list of all ARC buffers that are still valid on the
4678          * device.
4679          */
4680         adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
4681         list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
4682             offsetof(arc_buf_hdr_t, b_l2node));
4683 
4684         vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
4685 
4686         /*
4687          * Add device to global list
4688          */
4689         mutex_enter(&l2arc_dev_mtx);
4690         list_insert_head(l2arc_dev_list, adddev);
4691         atomic_inc_64(&l2arc_ndev);
4692         mutex_exit(&l2arc_dev_mtx);
4693 }
4694 
4695 /*
4696  * Remove a vdev from the L2ARC.
4697  */
4698 void
4699 l2arc_remove_vdev(vdev_t *vd)
4700 {
4701         l2arc_dev_t *dev, *nextdev, *remdev = NULL;
4702 
4703         /*
4704          * Find the device by vdev
4705          */
4706         mutex_enter(&l2arc_dev_mtx);
4707         for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
4708                 nextdev = list_next(l2arc_dev_list, dev);
4709                 if (vd == dev->l2ad_vdev) {
4710                         remdev = dev;
4711                         break;
4712                 }
4713         }
4714         ASSERT(remdev != NULL);
4715 
4716         /*
4717          * Remove device from global list
4718          */
4719         list_remove(l2arc_dev_list, remdev);
4720         l2arc_dev_last = NULL;          /* may have been invalidated */
4721         atomic_dec_64(&l2arc_ndev);
4722         mutex_exit(&l2arc_dev_mtx);
4723 
4724         /*
4725          * Clear all buflists and ARC references.  L2ARC device flush.
4726          */
4727         l2arc_evict(remdev, 0, B_TRUE);
4728         list_destroy(remdev->l2ad_buflist);
4729         kmem_free(remdev->l2ad_buflist, sizeof (list_t));
4730         kmem_free(remdev, sizeof (l2arc_dev_t));
4731 }
4732 
4733 void
4734 l2arc_init(void)
4735 {
4736         l2arc_thread_exit = 0;
4737         l2arc_ndev = 0;
4738         l2arc_writes_sent = 0;
4739         l2arc_writes_done = 0;
4740 
4741         mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
4742         cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
4743         mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
4744         mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
4745         mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
4746 
4747         l2arc_dev_list = &L2ARC_dev_list;
4748         l2arc_free_on_write = &L2ARC_free_on_write;
4749         list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
4750             offsetof(l2arc_dev_t, l2ad_node));
4751         list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
4752             offsetof(l2arc_data_free_t, l2df_list_node));
4753 }
4754 
4755 void
4756 l2arc_fini(void)
4757 {
4758         /*
4759          * This is called from dmu_fini(), which is called from spa_fini();
4760          * Because of this, we can assume that all l2arc devices have
4761          * already been removed when the pools themselves were removed.
4762          */
4763 
4764         l2arc_do_free_on_write();
4765 
4766         mutex_destroy(&l2arc_feed_thr_lock);
4767         cv_destroy(&l2arc_feed_thr_cv);
4768         mutex_destroy(&l2arc_dev_mtx);
4769         mutex_destroy(&l2arc_buflist_mtx);
4770         mutex_destroy(&l2arc_free_on_write_mtx);
4771 
4772         list_destroy(l2arc_dev_list);
4773         list_destroy(l2arc_free_on_write);
4774 }
4775 
4776 void
4777 l2arc_start(void)
4778 {
4779         if (!(spa_mode_global & FWRITE))
4780                 return;
4781 
4782         (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
4783             TS_RUN, minclsyspri);
4784 }
4785 
4786 void
4787 l2arc_stop(void)
4788 {
4789         if (!(spa_mode_global & FWRITE))
4790                 return;
4791 
4792         mutex_enter(&l2arc_feed_thr_lock);
4793         cv_signal(&l2arc_feed_thr_cv);      /* kick thread out of startup */
4794         l2arc_thread_exit = 1;
4795         while (l2arc_thread_exit != 0)
4796                 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
4797         mutex_exit(&l2arc_feed_thr_lock);
4798 }