1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  24  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  25  * Copyright (c) 2013 by Delphix. All rights reserved.
  26  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  27  */
  28 
  29 /*
  30  * DVA-based Adjustable Replacement Cache
  31  *
  32  * While much of the theory of operation used here is
  33  * based on the self-tuning, low overhead replacement cache
  34  * presented by Megiddo and Modha at FAST 2003, there are some
  35  * significant differences:
  36  *
  37  * 1. The Megiddo and Modha model assumes any page is evictable.
  38  * Pages in its cache cannot be "locked" into memory.  This makes
  39  * the eviction algorithm simple: evict the last page in the list.
  40  * This also make the performance characteristics easy to reason
  41  * about.  Our cache is not so simple.  At any given moment, some
  42  * subset of the blocks in the cache are un-evictable because we
  43  * have handed out a reference to them.  Blocks are only evictable
  44  * when there are no external references active.  This makes
  45  * eviction far more problematic:  we choose to evict the evictable
  46  * blocks that are the "lowest" in the list.
  47  *
  48  * There are times when it is not possible to evict the requested
  49  * space.  In these circumstances we are unable to adjust the cache
  50  * size.  To prevent the cache growing unbounded at these times we
  51  * implement a "cache throttle" that slows the flow of new data
  52  * into the cache until we can make space available.
  53  *
  54  * 2. The Megiddo and Modha model assumes a fixed cache size.
  55  * Pages are evicted when the cache is full and there is a cache
  56  * miss.  Our model has a variable sized cache.  It grows with
  57  * high use, but also tries to react to memory pressure from the
  58  * operating system: decreasing its size when system memory is
  59  * tight.
  60  *
  61  * 3. The Megiddo and Modha model assumes a fixed page size. All
  62  * elements of the cache are therefore exactly the same size.  So
  63  * when adjusting the cache size following a cache miss, its simply
  64  * a matter of choosing a single page to evict.  In our model, we
  65  * have variable sized cache blocks (rangeing from 512 bytes to
  66  * 128K bytes).  We therefore choose a set of blocks to evict to make
  67  * space for a cache miss that approximates as closely as possible
  68  * the space used by the new block.
  69  *
  70  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
  71  * by N. Megiddo & D. Modha, FAST 2003
  72  */
  73 
  74 /*
  75  * The locking model:
  76  *
  77  * A new reference to a cache buffer can be obtained in two
  78  * ways: 1) via a hash table lookup using the DVA as a key,
  79  * or 2) via one of the ARC lists.  The arc_read() interface
  80  * uses method 1, while the internal arc algorithms for
  81  * adjusting the cache use method 2.  We therefore provide two
  82  * types of locks: 1) the hash table lock array, and 2) the
  83  * arc list locks.
  84  *
  85  * Buffers do not have their own mutexes, rather they rely on the
  86  * hash table mutexes for the bulk of their protection (i.e. most
  87  * fields in the arc_buf_hdr_t are protected by these mutexes).
  88  *
  89  * buf_hash_find() returns the appropriate mutex (held) when it
  90  * locates the requested buffer in the hash table.  It returns
  91  * NULL for the mutex if the buffer was not in the table.
  92  *
  93  * buf_hash_remove() expects the appropriate hash mutex to be
  94  * already held before it is invoked.
  95  *
  96  * Each arc state also has a mutex which is used to protect the
  97  * buffer list associated with the state.  When attempting to
  98  * obtain a hash table lock while holding an arc list lock you
  99  * must use: mutex_tryenter() to avoid deadlock.  Also note that
 100  * the active state mutex must be held before the ghost state mutex.
 101  *
 102  * Arc buffers may have an associated eviction callback function.
 103  * This function will be invoked prior to removing the buffer (e.g.
 104  * in arc_do_user_evicts()).  Note however that the data associated
 105  * with the buffer may be evicted prior to the callback.  The callback
 106  * must be made with *no locks held* (to prevent deadlock).  Additionally,
 107  * the users of callbacks must ensure that their private data is
 108  * protected from simultaneous callbacks from arc_buf_evict()
 109  * and arc_do_user_evicts().
 110  *
 111  * Note that the majority of the performance stats are manipulated
 112  * with atomic operations.
 113  *
 114  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
 115  *
 116  *      - L2ARC buflist creation
 117  *      - L2ARC buflist eviction
 118  *      - L2ARC write completion, which walks L2ARC buflists
 119  *      - ARC header destruction, as it removes from L2ARC buflists
 120  *      - ARC header release, as it removes from L2ARC buflists
 121  */
 122 
 123 #include <sys/spa.h>
 124 #include <sys/zio.h>
 125 #include <sys/zio_compress.h>
 126 #include <sys/zfs_context.h>
 127 #include <sys/arc.h>
 128 #include <sys/refcount.h>
 129 #include <sys/vdev.h>
 130 #include <sys/vdev_impl.h>
 131 #include <sys/dsl_pool.h>
 132 #ifdef _KERNEL
 133 #include <sys/vmsystm.h>
 134 #include <vm/anon.h>
 135 #include <sys/fs/swapnode.h>
 136 #include <sys/dnlc.h>
 137 #endif
 138 #include <sys/callb.h>
 139 #include <sys/kstat.h>
 140 #include <zfs_fletcher.h>
 141 
 142 #ifndef _KERNEL
 143 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
 144 boolean_t arc_watch = B_FALSE;
 145 int arc_procfd;
 146 #endif
 147 
 148 static kmutex_t         arc_reclaim_thr_lock;
 149 static kcondvar_t       arc_reclaim_thr_cv;     /* used to signal reclaim thr */
 150 static uint8_t          arc_thread_exit;
 151 
 152 #define ARC_REDUCE_DNLC_PERCENT 3
 153 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
 154 
 155 typedef enum arc_reclaim_strategy {
 156         ARC_RECLAIM_AGGR,               /* Aggressive reclaim strategy */
 157         ARC_RECLAIM_CONS                /* Conservative reclaim strategy */
 158 } arc_reclaim_strategy_t;
 159 
 160 /*
 161  * The number of iterations through arc_evict_*() before we
 162  * drop & reacquire the lock.
 163  */
 164 int arc_evict_iterations = 100;
 165 
 166 /* number of seconds before growing cache again */
 167 static int              arc_grow_retry = 60;
 168 
 169 /* shift of arc_c for calculating both min and max arc_p */
 170 static int              arc_p_min_shift = 4;
 171 
 172 /* log2(fraction of arc to reclaim) */
 173 static int              arc_shrink_shift = 5;
 174 
 175 /*
 176  * minimum lifespan of a prefetch block in clock ticks
 177  * (initialized in arc_init())
 178  */
 179 static int              arc_min_prefetch_lifespan;
 180 
 181 /*
 182  * If this percent of memory is free, don't throttle.
 183  */
 184 int arc_lotsfree_percent = 10;
 185 
 186 static int arc_dead;
 187 
 188 /*
 189  * The arc has filled available memory and has now warmed up.
 190  */
 191 static boolean_t arc_warm;
 192 
 193 /*
 194  * These tunables are for performance analysis.
 195  */
 196 uint64_t zfs_arc_max;
 197 uint64_t zfs_arc_min;
 198 uint64_t zfs_arc_meta_limit = 0;
 199 int zfs_arc_grow_retry = 0;
 200 int zfs_arc_shrink_shift = 0;
 201 int zfs_arc_p_min_shift = 0;
 202 int zfs_disable_dup_eviction = 0;
 203 
 204 /*
 205  * Note that buffers can be in one of 6 states:
 206  *      ARC_anon        - anonymous (discussed below)
 207  *      ARC_mru         - recently used, currently cached
 208  *      ARC_mru_ghost   - recentely used, no longer in cache
 209  *      ARC_mfu         - frequently used, currently cached
 210  *      ARC_mfu_ghost   - frequently used, no longer in cache
 211  *      ARC_l2c_only    - exists in L2ARC but not other states
 212  * When there are no active references to the buffer, they are
 213  * are linked onto a list in one of these arc states.  These are
 214  * the only buffers that can be evicted or deleted.  Within each
 215  * state there are multiple lists, one for meta-data and one for
 216  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
 217  * etc.) is tracked separately so that it can be managed more
 218  * explicitly: favored over data, limited explicitly.
 219  *
 220  * Anonymous buffers are buffers that are not associated with
 221  * a DVA.  These are buffers that hold dirty block copies
 222  * before they are written to stable storage.  By definition,
 223  * they are "ref'd" and are considered part of arc_mru
 224  * that cannot be freed.  Generally, they will aquire a DVA
 225  * as they are written and migrate onto the arc_mru list.
 226  *
 227  * The ARC_l2c_only state is for buffers that are in the second
 228  * level ARC but no longer in any of the ARC_m* lists.  The second
 229  * level ARC itself may also contain buffers that are in any of
 230  * the ARC_m* states - meaning that a buffer can exist in two
 231  * places.  The reason for the ARC_l2c_only state is to keep the
 232  * buffer header in the hash table, so that reads that hit the
 233  * second level ARC benefit from these fast lookups.
 234  */
 235 
 236 typedef struct arc_state {
 237         list_t  arcs_list[ARC_BUFC_NUMTYPES];   /* list of evictable buffers */
 238         uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */
 239         uint64_t arcs_size;     /* total amount of data in this state */
 240         kmutex_t arcs_mtx;
 241 } arc_state_t;
 242 
 243 /* The 6 states: */
 244 static arc_state_t ARC_anon;
 245 static arc_state_t ARC_mru;
 246 static arc_state_t ARC_mru_ghost;
 247 static arc_state_t ARC_mfu;
 248 static arc_state_t ARC_mfu_ghost;
 249 static arc_state_t ARC_l2c_only;
 250 
 251 typedef struct arc_stats {
 252         kstat_named_t arcstat_hits;
 253         kstat_named_t arcstat_misses;
 254         kstat_named_t arcstat_demand_data_hits;
 255         kstat_named_t arcstat_demand_data_misses;
 256         kstat_named_t arcstat_demand_metadata_hits;
 257         kstat_named_t arcstat_demand_metadata_misses;
 258         kstat_named_t arcstat_prefetch_data_hits;
 259         kstat_named_t arcstat_prefetch_data_misses;
 260         kstat_named_t arcstat_prefetch_metadata_hits;
 261         kstat_named_t arcstat_prefetch_metadata_misses;
 262         kstat_named_t arcstat_mru_hits;
 263         kstat_named_t arcstat_mru_ghost_hits;
 264         kstat_named_t arcstat_mfu_hits;
 265         kstat_named_t arcstat_mfu_ghost_hits;
 266         kstat_named_t arcstat_deleted;
 267         kstat_named_t arcstat_recycle_miss;
 268         /*
 269          * Number of buffers that could not be evicted because the hash lock
 270          * was held by another thread.  The lock may not necessarily be held
 271          * by something using the same buffer, since hash locks are shared
 272          * by multiple buffers.
 273          */
 274         kstat_named_t arcstat_mutex_miss;
 275         /*
 276          * Number of buffers skipped because they have I/O in progress, are
 277          * indrect prefetch buffers that have not lived long enough, or are
 278          * not from the spa we're trying to evict from.
 279          */
 280         kstat_named_t arcstat_evict_skip;
 281         kstat_named_t arcstat_evict_l2_cached;
 282         kstat_named_t arcstat_evict_l2_eligible;
 283         kstat_named_t arcstat_evict_l2_ineligible;
 284         kstat_named_t arcstat_hash_elements;
 285         kstat_named_t arcstat_hash_elements_max;
 286         kstat_named_t arcstat_hash_collisions;
 287         kstat_named_t arcstat_hash_chains;
 288         kstat_named_t arcstat_hash_chain_max;
 289         kstat_named_t arcstat_p;
 290         kstat_named_t arcstat_c;
 291         kstat_named_t arcstat_c_min;
 292         kstat_named_t arcstat_c_max;
 293         kstat_named_t arcstat_size;
 294         kstat_named_t arcstat_hdr_size;
 295         kstat_named_t arcstat_data_size;
 296         kstat_named_t arcstat_other_size;
 297         kstat_named_t arcstat_l2_hits;
 298         kstat_named_t arcstat_l2_misses;
 299         kstat_named_t arcstat_l2_feeds;
 300         kstat_named_t arcstat_l2_rw_clash;
 301         kstat_named_t arcstat_l2_read_bytes;
 302         kstat_named_t arcstat_l2_write_bytes;
 303         kstat_named_t arcstat_l2_writes_sent;
 304         kstat_named_t arcstat_l2_writes_done;
 305         kstat_named_t arcstat_l2_writes_error;
 306         kstat_named_t arcstat_l2_writes_hdr_miss;
 307         kstat_named_t arcstat_l2_evict_lock_retry;
 308         kstat_named_t arcstat_l2_evict_reading;
 309         kstat_named_t arcstat_l2_free_on_write;
 310         kstat_named_t arcstat_l2_abort_lowmem;
 311         kstat_named_t arcstat_l2_cksum_bad;
 312         kstat_named_t arcstat_l2_io_error;
 313         kstat_named_t arcstat_l2_size;
 314         kstat_named_t arcstat_l2_asize;
 315         kstat_named_t arcstat_l2_hdr_size;
 316         kstat_named_t arcstat_l2_compress_successes;
 317         kstat_named_t arcstat_l2_compress_zeros;
 318         kstat_named_t arcstat_l2_compress_failures;
 319         kstat_named_t arcstat_memory_throttle_count;
 320         kstat_named_t arcstat_duplicate_buffers;
 321         kstat_named_t arcstat_duplicate_buffers_size;
 322         kstat_named_t arcstat_duplicate_reads;
 323         kstat_named_t arcstat_meta_used;
 324         kstat_named_t arcstat_meta_limit;
 325         kstat_named_t arcstat_meta_max;
 326 } arc_stats_t;
 327 
 328 static arc_stats_t arc_stats = {
 329         { "hits",                       KSTAT_DATA_UINT64 },
 330         { "misses",                     KSTAT_DATA_UINT64 },
 331         { "demand_data_hits",           KSTAT_DATA_UINT64 },
 332         { "demand_data_misses",         KSTAT_DATA_UINT64 },
 333         { "demand_metadata_hits",       KSTAT_DATA_UINT64 },
 334         { "demand_metadata_misses",     KSTAT_DATA_UINT64 },
 335         { "prefetch_data_hits",         KSTAT_DATA_UINT64 },
 336         { "prefetch_data_misses",       KSTAT_DATA_UINT64 },
 337         { "prefetch_metadata_hits",     KSTAT_DATA_UINT64 },
 338         { "prefetch_metadata_misses",   KSTAT_DATA_UINT64 },
 339         { "mru_hits",                   KSTAT_DATA_UINT64 },
 340         { "mru_ghost_hits",             KSTAT_DATA_UINT64 },
 341         { "mfu_hits",                   KSTAT_DATA_UINT64 },
 342         { "mfu_ghost_hits",             KSTAT_DATA_UINT64 },
 343         { "deleted",                    KSTAT_DATA_UINT64 },
 344         { "recycle_miss",               KSTAT_DATA_UINT64 },
 345         { "mutex_miss",                 KSTAT_DATA_UINT64 },
 346         { "evict_skip",                 KSTAT_DATA_UINT64 },
 347         { "evict_l2_cached",            KSTAT_DATA_UINT64 },
 348         { "evict_l2_eligible",          KSTAT_DATA_UINT64 },
 349         { "evict_l2_ineligible",        KSTAT_DATA_UINT64 },
 350         { "hash_elements",              KSTAT_DATA_UINT64 },
 351         { "hash_elements_max",          KSTAT_DATA_UINT64 },
 352         { "hash_collisions",            KSTAT_DATA_UINT64 },
 353         { "hash_chains",                KSTAT_DATA_UINT64 },
 354         { "hash_chain_max",             KSTAT_DATA_UINT64 },
 355         { "p",                          KSTAT_DATA_UINT64 },
 356         { "c",                          KSTAT_DATA_UINT64 },
 357         { "c_min",                      KSTAT_DATA_UINT64 },
 358         { "c_max",                      KSTAT_DATA_UINT64 },
 359         { "size",                       KSTAT_DATA_UINT64 },
 360         { "hdr_size",                   KSTAT_DATA_UINT64 },
 361         { "data_size",                  KSTAT_DATA_UINT64 },
 362         { "other_size",                 KSTAT_DATA_UINT64 },
 363         { "l2_hits",                    KSTAT_DATA_UINT64 },
 364         { "l2_misses",                  KSTAT_DATA_UINT64 },
 365         { "l2_feeds",                   KSTAT_DATA_UINT64 },
 366         { "l2_rw_clash",                KSTAT_DATA_UINT64 },
 367         { "l2_read_bytes",              KSTAT_DATA_UINT64 },
 368         { "l2_write_bytes",             KSTAT_DATA_UINT64 },
 369         { "l2_writes_sent",             KSTAT_DATA_UINT64 },
 370         { "l2_writes_done",             KSTAT_DATA_UINT64 },
 371         { "l2_writes_error",            KSTAT_DATA_UINT64 },
 372         { "l2_writes_hdr_miss",         KSTAT_DATA_UINT64 },
 373         { "l2_evict_lock_retry",        KSTAT_DATA_UINT64 },
 374         { "l2_evict_reading",           KSTAT_DATA_UINT64 },
 375         { "l2_free_on_write",           KSTAT_DATA_UINT64 },
 376         { "l2_abort_lowmem",            KSTAT_DATA_UINT64 },
 377         { "l2_cksum_bad",               KSTAT_DATA_UINT64 },
 378         { "l2_io_error",                KSTAT_DATA_UINT64 },
 379         { "l2_size",                    KSTAT_DATA_UINT64 },
 380         { "l2_asize",                   KSTAT_DATA_UINT64 },
 381         { "l2_hdr_size",                KSTAT_DATA_UINT64 },
 382         { "l2_compress_successes",      KSTAT_DATA_UINT64 },
 383         { "l2_compress_zeros",          KSTAT_DATA_UINT64 },
 384         { "l2_compress_failures",       KSTAT_DATA_UINT64 },
 385         { "memory_throttle_count",      KSTAT_DATA_UINT64 },
 386         { "duplicate_buffers",          KSTAT_DATA_UINT64 },
 387         { "duplicate_buffers_size",     KSTAT_DATA_UINT64 },
 388         { "duplicate_reads",            KSTAT_DATA_UINT64 },
 389         { "arc_meta_used",              KSTAT_DATA_UINT64 },
 390         { "arc_meta_limit",             KSTAT_DATA_UINT64 },
 391         { "arc_meta_max",               KSTAT_DATA_UINT64 }
 392 };
 393 
 394 #define ARCSTAT(stat)   (arc_stats.stat.value.ui64)
 395 
 396 #define ARCSTAT_INCR(stat, val) \
 397         atomic_add_64(&arc_stats.stat.value.ui64, (val))
 398 
 399 #define ARCSTAT_BUMP(stat)      ARCSTAT_INCR(stat, 1)
 400 #define ARCSTAT_BUMPDOWN(stat)  ARCSTAT_INCR(stat, -1)
 401 
 402 #define ARCSTAT_MAX(stat, val) {                                        \
 403         uint64_t m;                                                     \
 404         while ((val) > (m = arc_stats.stat.value.ui64) &&            \
 405             (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))     \
 406                 continue;                                               \
 407 }
 408 
 409 #define ARCSTAT_MAXSTAT(stat) \
 410         ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
 411 
 412 /*
 413  * We define a macro to allow ARC hits/misses to be easily broken down by
 414  * two separate conditions, giving a total of four different subtypes for
 415  * each of hits and misses (so eight statistics total).
 416  */
 417 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
 418         if (cond1) {                                                    \
 419                 if (cond2) {                                            \
 420                         ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
 421                 } else {                                                \
 422                         ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
 423                 }                                                       \
 424         } else {                                                        \
 425                 if (cond2) {                                            \
 426                         ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
 427                 } else {                                                \
 428                         ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
 429                 }                                                       \
 430         }
 431 
 432 kstat_t                 *arc_ksp;
 433 static arc_state_t      *arc_anon;
 434 static arc_state_t      *arc_mru;
 435 static arc_state_t      *arc_mru_ghost;
 436 static arc_state_t      *arc_mfu;
 437 static arc_state_t      *arc_mfu_ghost;
 438 static arc_state_t      *arc_l2c_only;
 439 
 440 /*
 441  * There are several ARC variables that are critical to export as kstats --
 442  * but we don't want to have to grovel around in the kstat whenever we wish to
 443  * manipulate them.  For these variables, we therefore define them to be in
 444  * terms of the statistic variable.  This assures that we are not introducing
 445  * the possibility of inconsistency by having shadow copies of the variables,
 446  * while still allowing the code to be readable.
 447  */
 448 #define arc_size        ARCSTAT(arcstat_size)   /* actual total arc size */
 449 #define arc_p           ARCSTAT(arcstat_p)      /* target size of MRU */
 450 #define arc_c           ARCSTAT(arcstat_c)      /* target size of cache */
 451 #define arc_c_min       ARCSTAT(arcstat_c_min)  /* min target cache size */
 452 #define arc_c_max       ARCSTAT(arcstat_c_max)  /* max target cache size */
 453 #define arc_meta_limit  ARCSTAT(arcstat_meta_limit) /* max size for metadata */
 454 #define arc_meta_used   ARCSTAT(arcstat_meta_used) /* size of metadata */
 455 #define arc_meta_max    ARCSTAT(arcstat_meta_max) /* max size of metadata */
 456 
 457 #define L2ARC_IS_VALID_COMPRESS(_c_) \
 458         ((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
 459 
 460 static int              arc_no_grow;    /* Don't try to grow cache size */
 461 static uint64_t         arc_tempreserve;
 462 static uint64_t         arc_loaned_bytes;
 463 
 464 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
 465 
 466 typedef struct arc_callback arc_callback_t;
 467 
 468 struct arc_callback {
 469         void                    *acb_private;
 470         arc_done_func_t         *acb_done;
 471         arc_buf_t               *acb_buf;
 472         zio_t                   *acb_zio_dummy;
 473         arc_callback_t          *acb_next;
 474 };
 475 
 476 typedef struct arc_write_callback arc_write_callback_t;
 477 
 478 struct arc_write_callback {
 479         void            *awcb_private;
 480         arc_done_func_t *awcb_ready;
 481         arc_done_func_t *awcb_physdone;
 482         arc_done_func_t *awcb_done;
 483         arc_buf_t       *awcb_buf;
 484 };
 485 
 486 struct arc_buf_hdr {
 487         /* protected by hash lock */
 488         dva_t                   b_dva;
 489         uint64_t                b_birth;
 490         uint64_t                b_cksum0;
 491 
 492         kmutex_t                b_freeze_lock;
 493         zio_cksum_t             *b_freeze_cksum;
 494         void                    *b_thawed;
 495 
 496         arc_buf_hdr_t           *b_hash_next;
 497         arc_buf_t               *b_buf;
 498         uint32_t                b_flags;
 499         uint32_t                b_datacnt;
 500 
 501         arc_callback_t          *b_acb;
 502         kcondvar_t              b_cv;
 503 
 504         /* immutable */
 505         arc_buf_contents_t      b_type;
 506         uint64_t                b_size;
 507         uint64_t                b_spa;
 508 
 509         /* protected by arc state mutex */
 510         arc_state_t             *b_state;
 511         list_node_t             b_arc_node;
 512 
 513         /* updated atomically */
 514         clock_t                 b_arc_access;
 515 
 516         /* self protecting */
 517         refcount_t              b_refcnt;
 518 
 519         l2arc_buf_hdr_t         *b_l2hdr;
 520         list_node_t             b_l2node;
 521 };
 522 
 523 static arc_buf_t *arc_eviction_list;
 524 static kmutex_t arc_eviction_mtx;
 525 static arc_buf_hdr_t arc_eviction_hdr;
 526 static void arc_get_data_buf(arc_buf_t *buf);
 527 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
 528 static int arc_evict_needed(arc_buf_contents_t type);
 529 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
 530 static void arc_buf_watch(arc_buf_t *buf);
 531 
 532 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
 533 
 534 #define GHOST_STATE(state)      \
 535         ((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||        \
 536         (state) == arc_l2c_only)
 537 
 538 /*
 539  * Private ARC flags.  These flags are private ARC only flags that will show up
 540  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
 541  * be passed in as arc_flags in things like arc_read.  However, these flags
 542  * should never be passed and should only be set by ARC code.  When adding new
 543  * public flags, make sure not to smash the private ones.
 544  */
 545 
 546 #define ARC_IN_HASH_TABLE       (1 << 9)  /* this buffer is hashed */
 547 #define ARC_IO_IN_PROGRESS      (1 << 10) /* I/O in progress for buf */
 548 #define ARC_IO_ERROR            (1 << 11) /* I/O failed for buf */
 549 #define ARC_FREED_IN_READ       (1 << 12) /* buf freed while in read */
 550 #define ARC_BUF_AVAILABLE       (1 << 13) /* block not in active use */
 551 #define ARC_INDIRECT            (1 << 14) /* this is an indirect block */
 552 #define ARC_FREE_IN_PROGRESS    (1 << 15) /* hdr about to be freed */
 553 #define ARC_L2_WRITING          (1 << 16) /* L2ARC write in progress */
 554 #define ARC_L2_EVICTED          (1 << 17) /* evicted during I/O */
 555 #define ARC_L2_WRITE_HEAD       (1 << 18) /* head of write list */
 556 
 557 #define HDR_IN_HASH_TABLE(hdr)  ((hdr)->b_flags & ARC_IN_HASH_TABLE)
 558 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS)
 559 #define HDR_IO_ERROR(hdr)       ((hdr)->b_flags & ARC_IO_ERROR)
 560 #define HDR_PREFETCH(hdr)       ((hdr)->b_flags & ARC_PREFETCH)
 561 #define HDR_FREED_IN_READ(hdr)  ((hdr)->b_flags & ARC_FREED_IN_READ)
 562 #define HDR_BUF_AVAILABLE(hdr)  ((hdr)->b_flags & ARC_BUF_AVAILABLE)
 563 #define HDR_FREE_IN_PROGRESS(hdr)       ((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
 564 #define HDR_L2CACHE(hdr)        ((hdr)->b_flags & ARC_L2CACHE)
 565 #define HDR_L2_READING(hdr)     ((hdr)->b_flags & ARC_IO_IN_PROGRESS &&  \
 566                                     (hdr)->b_l2hdr != NULL)
 567 #define HDR_L2_WRITING(hdr)     ((hdr)->b_flags & ARC_L2_WRITING)
 568 #define HDR_L2_EVICTED(hdr)     ((hdr)->b_flags & ARC_L2_EVICTED)
 569 #define HDR_L2_WRITE_HEAD(hdr)  ((hdr)->b_flags & ARC_L2_WRITE_HEAD)
 570 
 571 /*
 572  * Other sizes
 573  */
 574 
 575 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
 576 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
 577 
 578 /*
 579  * Hash table routines
 580  */
 581 
 582 #define HT_LOCK_PAD     64
 583 
 584 struct ht_lock {
 585         kmutex_t        ht_lock;
 586 #ifdef _KERNEL
 587         unsigned char   pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
 588 #endif
 589 };
 590 
 591 #define BUF_LOCKS 256
 592 typedef struct buf_hash_table {
 593         uint64_t ht_mask;
 594         arc_buf_hdr_t **ht_table;
 595         struct ht_lock ht_locks[BUF_LOCKS];
 596 } buf_hash_table_t;
 597 
 598 static buf_hash_table_t buf_hash_table;
 599 
 600 #define BUF_HASH_INDEX(spa, dva, birth) \
 601         (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
 602 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
 603 #define BUF_HASH_LOCK(idx)      (&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
 604 #define HDR_LOCK(hdr) \
 605         (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
 606 
 607 uint64_t zfs_crc64_table[256];
 608 
 609 /*
 610  * Level 2 ARC
 611  */
 612 
 613 #define L2ARC_WRITE_SIZE        (8 * 1024 * 1024)       /* initial write max */
 614 #define L2ARC_HEADROOM          2                       /* num of writes */
 615 /*
 616  * If we discover during ARC scan any buffers to be compressed, we boost
 617  * our headroom for the next scanning cycle by this percentage multiple.
 618  */
 619 #define L2ARC_HEADROOM_BOOST    200
 620 #define L2ARC_FEED_SECS         1               /* caching interval secs */
 621 #define L2ARC_FEED_MIN_MS       200             /* min caching interval ms */
 622 
 623 #define l2arc_writes_sent       ARCSTAT(arcstat_l2_writes_sent)
 624 #define l2arc_writes_done       ARCSTAT(arcstat_l2_writes_done)
 625 
 626 /* L2ARC Performance Tunables */
 627 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;    /* default max write size */
 628 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;  /* extra write during warmup */
 629 uint64_t l2arc_headroom = L2ARC_HEADROOM;       /* number of dev writes */
 630 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
 631 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;     /* interval seconds */
 632 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
 633 boolean_t l2arc_noprefetch = B_TRUE;            /* don't cache prefetch bufs */
 634 boolean_t l2arc_feed_again = B_TRUE;            /* turbo warmup */
 635 boolean_t l2arc_norw = B_TRUE;                  /* no reads during writes */
 636 
 637 /*
 638  * L2ARC Internals
 639  */
 640 typedef struct l2arc_dev {
 641         vdev_t                  *l2ad_vdev;     /* vdev */
 642         spa_t                   *l2ad_spa;      /* spa */
 643         uint64_t                l2ad_hand;      /* next write location */
 644         uint64_t                l2ad_start;     /* first addr on device */
 645         uint64_t                l2ad_end;       /* last addr on device */
 646         uint64_t                l2ad_evict;     /* last addr eviction reached */
 647         boolean_t               l2ad_first;     /* first sweep through */
 648         boolean_t               l2ad_writing;   /* currently writing */
 649         list_t                  *l2ad_buflist;  /* buffer list */
 650         list_node_t             l2ad_node;      /* device list node */
 651 } l2arc_dev_t;
 652 
 653 static list_t L2ARC_dev_list;                   /* device list */
 654 static list_t *l2arc_dev_list;                  /* device list pointer */
 655 static kmutex_t l2arc_dev_mtx;                  /* device list mutex */
 656 static l2arc_dev_t *l2arc_dev_last;             /* last device used */
 657 static kmutex_t l2arc_buflist_mtx;              /* mutex for all buflists */
 658 static list_t L2ARC_free_on_write;              /* free after write buf list */
 659 static list_t *l2arc_free_on_write;             /* free after write list ptr */
 660 static kmutex_t l2arc_free_on_write_mtx;        /* mutex for list */
 661 static uint64_t l2arc_ndev;                     /* number of devices */
 662 
 663 typedef struct l2arc_read_callback {
 664         arc_buf_t               *l2rcb_buf;             /* read buffer */
 665         spa_t                   *l2rcb_spa;             /* spa */
 666         blkptr_t                l2rcb_bp;               /* original blkptr */
 667         zbookmark_t             l2rcb_zb;               /* original bookmark */
 668         int                     l2rcb_flags;            /* original flags */
 669         enum zio_compress       l2rcb_compress;         /* applied compress */
 670 } l2arc_read_callback_t;
 671 
 672 typedef struct l2arc_write_callback {
 673         l2arc_dev_t     *l2wcb_dev;             /* device info */
 674         arc_buf_hdr_t   *l2wcb_head;            /* head of write buflist */
 675 } l2arc_write_callback_t;
 676 
 677 struct l2arc_buf_hdr {
 678         /* protected by arc_buf_hdr  mutex */
 679         l2arc_dev_t             *b_dev;         /* L2ARC device */
 680         uint64_t                b_daddr;        /* disk address, offset byte */
 681         /* compression applied to buffer data */
 682         enum zio_compress       b_compress;
 683         /* real alloc'd buffer size depending on b_compress applied */
 684         int                     b_asize;
 685         /* temporary buffer holder for in-flight compressed data */
 686         void                    *b_tmp_cdata;
 687 };
 688 
 689 typedef struct l2arc_data_free {
 690         /* protected by l2arc_free_on_write_mtx */
 691         void            *l2df_data;
 692         size_t          l2df_size;
 693         void            (*l2df_func)(void *, size_t);
 694         list_node_t     l2df_list_node;
 695 } l2arc_data_free_t;
 696 
 697 static kmutex_t l2arc_feed_thr_lock;
 698 static kcondvar_t l2arc_feed_thr_cv;
 699 static uint8_t l2arc_thread_exit;
 700 
 701 static void l2arc_read_done(zio_t *zio);
 702 static void l2arc_hdr_stat_add(void);
 703 static void l2arc_hdr_stat_remove(void);
 704 
 705 static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr);
 706 static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr,
 707     enum zio_compress c);
 708 static void l2arc_release_cdata_buf(arc_buf_hdr_t *ab);
 709 
 710 static uint64_t
 711 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 712 {
 713         uint8_t *vdva = (uint8_t *)dva;
 714         uint64_t crc = -1ULL;
 715         int i;
 716 
 717         ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
 718 
 719         for (i = 0; i < sizeof (dva_t); i++)
 720                 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
 721 
 722         crc ^= (spa>>8) ^ birth;
 723 
 724         return (crc);
 725 }
 726 
 727 #define BUF_EMPTY(buf)                                          \
 728         ((buf)->b_dva.dva_word[0] == 0 &&                    \
 729         (buf)->b_dva.dva_word[1] == 0 &&                     \
 730         (buf)->b_birth == 0)
 731 
 732 #define BUF_EQUAL(spa, dva, birth, buf)                         \
 733         ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&       \
 734         ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&       \
 735         ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
 736 
 737 static void
 738 buf_discard_identity(arc_buf_hdr_t *hdr)
 739 {
 740         hdr->b_dva.dva_word[0] = 0;
 741         hdr->b_dva.dva_word[1] = 0;
 742         hdr->b_birth = 0;
 743         hdr->b_cksum0 = 0;
 744 }
 745 
 746 static arc_buf_hdr_t *
 747 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
 748 {
 749         uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 750         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 751         arc_buf_hdr_t *buf;
 752 
 753         mutex_enter(hash_lock);
 754         for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
 755             buf = buf->b_hash_next) {
 756                 if (BUF_EQUAL(spa, dva, birth, buf)) {
 757                         *lockp = hash_lock;
 758                         return (buf);
 759                 }
 760         }
 761         mutex_exit(hash_lock);
 762         *lockp = NULL;
 763         return (NULL);
 764 }
 765 
 766 /*
 767  * Insert an entry into the hash table.  If there is already an element
 768  * equal to elem in the hash table, then the already existing element
 769  * will be returned and the new element will not be inserted.
 770  * Otherwise returns NULL.
 771  */
 772 static arc_buf_hdr_t *
 773 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
 774 {
 775         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 776         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 777         arc_buf_hdr_t *fbuf;
 778         uint32_t i;
 779 
 780         ASSERT(!HDR_IN_HASH_TABLE(buf));
 781         *lockp = hash_lock;
 782         mutex_enter(hash_lock);
 783         for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
 784             fbuf = fbuf->b_hash_next, i++) {
 785                 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
 786                         return (fbuf);
 787         }
 788 
 789         buf->b_hash_next = buf_hash_table.ht_table[idx];
 790         buf_hash_table.ht_table[idx] = buf;
 791         buf->b_flags |= ARC_IN_HASH_TABLE;
 792 
 793         /* collect some hash table performance data */
 794         if (i > 0) {
 795                 ARCSTAT_BUMP(arcstat_hash_collisions);
 796                 if (i == 1)
 797                         ARCSTAT_BUMP(arcstat_hash_chains);
 798 
 799                 ARCSTAT_MAX(arcstat_hash_chain_max, i);
 800         }
 801 
 802         ARCSTAT_BUMP(arcstat_hash_elements);
 803         ARCSTAT_MAXSTAT(arcstat_hash_elements);
 804 
 805         return (NULL);
 806 }
 807 
 808 static void
 809 buf_hash_remove(arc_buf_hdr_t *buf)
 810 {
 811         arc_buf_hdr_t *fbuf, **bufp;
 812         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 813 
 814         ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
 815         ASSERT(HDR_IN_HASH_TABLE(buf));
 816 
 817         bufp = &buf_hash_table.ht_table[idx];
 818         while ((fbuf = *bufp) != buf) {
 819                 ASSERT(fbuf != NULL);
 820                 bufp = &fbuf->b_hash_next;
 821         }
 822         *bufp = buf->b_hash_next;
 823         buf->b_hash_next = NULL;
 824         buf->b_flags &= ~ARC_IN_HASH_TABLE;
 825 
 826         /* collect some hash table performance data */
 827         ARCSTAT_BUMPDOWN(arcstat_hash_elements);
 828 
 829         if (buf_hash_table.ht_table[idx] &&
 830             buf_hash_table.ht_table[idx]->b_hash_next == NULL)
 831                 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
 832 }
 833 
 834 /*
 835  * Global data structures and functions for the buf kmem cache.
 836  */
 837 static kmem_cache_t *hdr_cache;
 838 static kmem_cache_t *buf_cache;
 839 
 840 static void
 841 buf_fini(void)
 842 {
 843         int i;
 844 
 845         kmem_free(buf_hash_table.ht_table,
 846             (buf_hash_table.ht_mask + 1) * sizeof (void *));
 847         for (i = 0; i < BUF_LOCKS; i++)
 848                 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
 849         kmem_cache_destroy(hdr_cache);
 850         kmem_cache_destroy(buf_cache);
 851 }
 852 
 853 /*
 854  * Constructor callback - called when the cache is empty
 855  * and a new buf is requested.
 856  */
 857 /* ARGSUSED */
 858 static int
 859 hdr_cons(void *vbuf, void *unused, int kmflag)
 860 {
 861         arc_buf_hdr_t *buf = vbuf;
 862 
 863         bzero(buf, sizeof (arc_buf_hdr_t));
 864         refcount_create(&buf->b_refcnt);
 865         cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
 866         mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
 867         arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 868 
 869         return (0);
 870 }
 871 
 872 /* ARGSUSED */
 873 static int
 874 buf_cons(void *vbuf, void *unused, int kmflag)
 875 {
 876         arc_buf_t *buf = vbuf;
 877 
 878         bzero(buf, sizeof (arc_buf_t));
 879         mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
 880         arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 881 
 882         return (0);
 883 }
 884 
 885 /*
 886  * Destructor callback - called when a cached buf is
 887  * no longer required.
 888  */
 889 /* ARGSUSED */
 890 static void
 891 hdr_dest(void *vbuf, void *unused)
 892 {
 893         arc_buf_hdr_t *buf = vbuf;
 894 
 895         ASSERT(BUF_EMPTY(buf));
 896         refcount_destroy(&buf->b_refcnt);
 897         cv_destroy(&buf->b_cv);
 898         mutex_destroy(&buf->b_freeze_lock);
 899         arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 900 }
 901 
 902 /* ARGSUSED */
 903 static void
 904 buf_dest(void *vbuf, void *unused)
 905 {
 906         arc_buf_t *buf = vbuf;
 907 
 908         mutex_destroy(&buf->b_evict_lock);
 909         arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 910 }
 911 
 912 /*
 913  * Reclaim callback -- invoked when memory is low.
 914  */
 915 /* ARGSUSED */
 916 static void
 917 hdr_recl(void *unused)
 918 {
 919         dprintf("hdr_recl called\n");
 920         /*
 921          * umem calls the reclaim func when we destroy the buf cache,
 922          * which is after we do arc_fini().
 923          */
 924         if (!arc_dead)
 925                 cv_signal(&arc_reclaim_thr_cv);
 926 }
 927 
 928 static void
 929 buf_init(void)
 930 {
 931         uint64_t *ct;
 932         uint64_t hsize = 1ULL << 12;
 933         int i, j;
 934 
 935         /*
 936          * The hash table is big enough to fill all of physical memory
 937          * with an average 64K block size.  The table will take up
 938          * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
 939          */
 940         while (hsize * 65536 < physmem * PAGESIZE)
 941                 hsize <<= 1;
 942 retry:
 943         buf_hash_table.ht_mask = hsize - 1;
 944         buf_hash_table.ht_table =
 945             kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
 946         if (buf_hash_table.ht_table == NULL) {
 947                 ASSERT(hsize > (1ULL << 8));
 948                 hsize >>= 1;
 949                 goto retry;
 950         }
 951 
 952         hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
 953             0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
 954         buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 955             0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 956 
 957         for (i = 0; i < 256; i++)
 958                 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
 959                         *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
 960 
 961         for (i = 0; i < BUF_LOCKS; i++) {
 962                 mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
 963                     NULL, MUTEX_DEFAULT, NULL);
 964         }
 965 }
 966 
 967 #define ARC_MINTIME     (hz>>4) /* 62 ms */
 968 
 969 static void
 970 arc_cksum_verify(arc_buf_t *buf)
 971 {
 972         zio_cksum_t zc;
 973 
 974         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 975                 return;
 976 
 977         mutex_enter(&buf->b_hdr->b_freeze_lock);
 978         if (buf->b_hdr->b_freeze_cksum == NULL ||
 979             (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
 980                 mutex_exit(&buf->b_hdr->b_freeze_lock);
 981                 return;
 982         }
 983         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 984         if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 985                 panic("buffer modified while frozen!");
 986         mutex_exit(&buf->b_hdr->b_freeze_lock);
 987 }
 988 
 989 static int
 990 arc_cksum_equal(arc_buf_t *buf)
 991 {
 992         zio_cksum_t zc;
 993         int equal;
 994 
 995         mutex_enter(&buf->b_hdr->b_freeze_lock);
 996         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 997         equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
 998         mutex_exit(&buf->b_hdr->b_freeze_lock);
 999 
1000         return (equal);
1001 }
1002 
1003 static void
1004 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1005 {
1006         if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1007                 return;
1008 
1009         mutex_enter(&buf->b_hdr->b_freeze_lock);
1010         if (buf->b_hdr->b_freeze_cksum != NULL) {
1011                 mutex_exit(&buf->b_hdr->b_freeze_lock);
1012                 return;
1013         }
1014         buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1015         fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1016             buf->b_hdr->b_freeze_cksum);
1017         mutex_exit(&buf->b_hdr->b_freeze_lock);
1018         arc_buf_watch(buf);
1019 }
1020 
1021 #ifndef _KERNEL
1022 typedef struct procctl {
1023         long cmd;
1024         prwatch_t prwatch;
1025 } procctl_t;
1026 #endif
1027 
1028 /* ARGSUSED */
1029 static void
1030 arc_buf_unwatch(arc_buf_t *buf)
1031 {
1032 #ifndef _KERNEL
1033         if (arc_watch) {
1034                 int result;
1035                 procctl_t ctl;
1036                 ctl.cmd = PCWATCH;
1037                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1038                 ctl.prwatch.pr_size = 0;
1039                 ctl.prwatch.pr_wflags = 0;
1040                 result = write(arc_procfd, &ctl, sizeof (ctl));
1041                 ASSERT3U(result, ==, sizeof (ctl));
1042         }
1043 #endif
1044 }
1045 
1046 /* ARGSUSED */
1047 static void
1048 arc_buf_watch(arc_buf_t *buf)
1049 {
1050 #ifndef _KERNEL
1051         if (arc_watch) {
1052                 int result;
1053                 procctl_t ctl;
1054                 ctl.cmd = PCWATCH;
1055                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1056                 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1057                 ctl.prwatch.pr_wflags = WA_WRITE;
1058                 result = write(arc_procfd, &ctl, sizeof (ctl));
1059                 ASSERT3U(result, ==, sizeof (ctl));
1060         }
1061 #endif
1062 }
1063 
1064 void
1065 arc_buf_thaw(arc_buf_t *buf)
1066 {
1067         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1068                 if (buf->b_hdr->b_state != arc_anon)
1069                         panic("modifying non-anon buffer!");
1070                 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1071                         panic("modifying buffer while i/o in progress!");
1072                 arc_cksum_verify(buf);
1073         }
1074 
1075         mutex_enter(&buf->b_hdr->b_freeze_lock);
1076         if (buf->b_hdr->b_freeze_cksum != NULL) {
1077                 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1078                 buf->b_hdr->b_freeze_cksum = NULL;
1079         }
1080 
1081         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1082                 if (buf->b_hdr->b_thawed)
1083                         kmem_free(buf->b_hdr->b_thawed, 1);
1084                 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1085         }
1086 
1087         mutex_exit(&buf->b_hdr->b_freeze_lock);
1088 
1089         arc_buf_unwatch(buf);
1090 }
1091 
1092 void
1093 arc_buf_freeze(arc_buf_t *buf)
1094 {
1095         kmutex_t *hash_lock;
1096 
1097         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1098                 return;
1099 
1100         hash_lock = HDR_LOCK(buf->b_hdr);
1101         mutex_enter(hash_lock);
1102 
1103         ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1104             buf->b_hdr->b_state == arc_anon);
1105         arc_cksum_compute(buf, B_FALSE);
1106         mutex_exit(hash_lock);
1107 
1108 }
1109 
1110 static void
1111 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1112 {
1113         ASSERT(MUTEX_HELD(hash_lock));
1114 
1115         if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
1116             (ab->b_state != arc_anon)) {
1117                 uint64_t delta = ab->b_size * ab->b_datacnt;
1118                 list_t *list = &ab->b_state->arcs_list[ab->b_type];
1119                 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
1120 
1121                 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
1122                 mutex_enter(&ab->b_state->arcs_mtx);
1123                 ASSERT(list_link_active(&ab->b_arc_node));
1124                 list_remove(list, ab);
1125                 if (GHOST_STATE(ab->b_state)) {
1126                         ASSERT0(ab->b_datacnt);
1127                         ASSERT3P(ab->b_buf, ==, NULL);
1128                         delta = ab->b_size;
1129                 }
1130                 ASSERT(delta > 0);
1131                 ASSERT3U(*size, >=, delta);
1132                 atomic_add_64(size, -delta);
1133                 mutex_exit(&ab->b_state->arcs_mtx);
1134                 /* remove the prefetch flag if we get a reference */
1135                 if (ab->b_flags & ARC_PREFETCH)
1136                         ab->b_flags &= ~ARC_PREFETCH;
1137         }
1138 }
1139 
1140 static int
1141 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1142 {
1143         int cnt;
1144         arc_state_t *state = ab->b_state;
1145 
1146         ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1147         ASSERT(!GHOST_STATE(state));
1148 
1149         if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
1150             (state != arc_anon)) {
1151                 uint64_t *size = &state->arcs_lsize[ab->b_type];
1152 
1153                 ASSERT(!MUTEX_HELD(&state->arcs_mtx));
1154                 mutex_enter(&state->arcs_mtx);
1155                 ASSERT(!list_link_active(&ab->b_arc_node));
1156                 list_insert_head(&state->arcs_list[ab->b_type], ab);
1157                 ASSERT(ab->b_datacnt > 0);
1158                 atomic_add_64(size, ab->b_size * ab->b_datacnt);
1159                 mutex_exit(&state->arcs_mtx);
1160         }
1161         return (cnt);
1162 }
1163 
1164 /*
1165  * Move the supplied buffer to the indicated state.  The mutex
1166  * for the buffer must be held by the caller.
1167  */
1168 static void
1169 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1170 {
1171         arc_state_t *old_state = ab->b_state;
1172         int64_t refcnt = refcount_count(&ab->b_refcnt);
1173         uint64_t from_delta, to_delta;
1174 
1175         ASSERT(MUTEX_HELD(hash_lock));
1176         ASSERT3P(new_state, !=, old_state);
1177         ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1178         ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1179         ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1180 
1181         from_delta = to_delta = ab->b_datacnt * ab->b_size;
1182 
1183         /*
1184          * If this buffer is evictable, transfer it from the
1185          * old state list to the new state list.
1186          */
1187         if (refcnt == 0) {
1188                 if (old_state != arc_anon) {
1189                         int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
1190                         uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1191 
1192                         if (use_mutex)
1193                                 mutex_enter(&old_state->arcs_mtx);
1194 
1195                         ASSERT(list_link_active(&ab->b_arc_node));
1196                         list_remove(&old_state->arcs_list[ab->b_type], ab);
1197 
1198                         /*
1199                          * If prefetching out of the ghost cache,
1200                          * we will have a non-zero datacnt.
1201                          */
1202                         if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
1203                                 /* ghost elements have a ghost size */
1204                                 ASSERT(ab->b_buf == NULL);
1205                                 from_delta = ab->b_size;
1206                         }
1207                         ASSERT3U(*size, >=, from_delta);
1208                         atomic_add_64(size, -from_delta);
1209 
1210                         if (use_mutex)
1211                                 mutex_exit(&old_state->arcs_mtx);
1212                 }
1213                 if (new_state != arc_anon) {
1214                         int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
1215                         uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1216 
1217                         if (use_mutex)
1218                                 mutex_enter(&new_state->arcs_mtx);
1219 
1220                         list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1221 
1222                         /* ghost elements have a ghost size */
1223                         if (GHOST_STATE(new_state)) {
1224                                 ASSERT(ab->b_datacnt == 0);
1225                                 ASSERT(ab->b_buf == NULL);
1226                                 to_delta = ab->b_size;
1227                         }
1228                         atomic_add_64(size, to_delta);
1229 
1230                         if (use_mutex)
1231                                 mutex_exit(&new_state->arcs_mtx);
1232                 }
1233         }
1234 
1235         ASSERT(!BUF_EMPTY(ab));
1236         if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
1237                 buf_hash_remove(ab);
1238 
1239         /* adjust state sizes */
1240         if (to_delta)
1241                 atomic_add_64(&new_state->arcs_size, to_delta);
1242         if (from_delta) {
1243                 ASSERT3U(old_state->arcs_size, >=, from_delta);
1244                 atomic_add_64(&old_state->arcs_size, -from_delta);
1245         }
1246         ab->b_state = new_state;
1247 
1248         /* adjust l2arc hdr stats */
1249         if (new_state == arc_l2c_only)
1250                 l2arc_hdr_stat_add();
1251         else if (old_state == arc_l2c_only)
1252                 l2arc_hdr_stat_remove();
1253 }
1254 
1255 void
1256 arc_space_consume(uint64_t space, arc_space_type_t type)
1257 {
1258         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1259 
1260         switch (type) {
1261         case ARC_SPACE_DATA:
1262                 ARCSTAT_INCR(arcstat_data_size, space);
1263                 break;
1264         case ARC_SPACE_OTHER:
1265                 ARCSTAT_INCR(arcstat_other_size, space);
1266                 break;
1267         case ARC_SPACE_HDRS:
1268                 ARCSTAT_INCR(arcstat_hdr_size, space);
1269                 break;
1270         case ARC_SPACE_L2HDRS:
1271                 ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1272                 break;
1273         }
1274 
1275         ARCSTAT_INCR(arcstat_meta_used, space);
1276         atomic_add_64(&arc_size, space);
1277 }
1278 
1279 void
1280 arc_space_return(uint64_t space, arc_space_type_t type)
1281 {
1282         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1283 
1284         switch (type) {
1285         case ARC_SPACE_DATA:
1286                 ARCSTAT_INCR(arcstat_data_size, -space);
1287                 break;
1288         case ARC_SPACE_OTHER:
1289                 ARCSTAT_INCR(arcstat_other_size, -space);
1290                 break;
1291         case ARC_SPACE_HDRS:
1292                 ARCSTAT_INCR(arcstat_hdr_size, -space);
1293                 break;
1294         case ARC_SPACE_L2HDRS:
1295                 ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1296                 break;
1297         }
1298 
1299         ASSERT(arc_meta_used >= space);
1300         if (arc_meta_max < arc_meta_used)
1301                 arc_meta_max = arc_meta_used;
1302         ARCSTAT_INCR(arcstat_meta_used, -space);
1303         ASSERT(arc_size >= space);
1304         atomic_add_64(&arc_size, -space);
1305 }
1306 
1307 void *
1308 arc_data_buf_alloc(uint64_t size)
1309 {
1310         if (arc_evict_needed(ARC_BUFC_DATA))
1311                 cv_signal(&arc_reclaim_thr_cv);
1312         atomic_add_64(&arc_size, size);
1313         return (zio_data_buf_alloc(size));
1314 }
1315 
1316 void
1317 arc_data_buf_free(void *buf, uint64_t size)
1318 {
1319         zio_data_buf_free(buf, size);
1320         ASSERT(arc_size >= size);
1321         atomic_add_64(&arc_size, -size);
1322 }
1323 
1324 arc_buf_t *
1325 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1326 {
1327         arc_buf_hdr_t *hdr;
1328         arc_buf_t *buf;
1329 
1330         ASSERT3U(size, >, 0);
1331         hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1332         ASSERT(BUF_EMPTY(hdr));
1333         hdr->b_size = size;
1334         hdr->b_type = type;
1335         hdr->b_spa = spa_load_guid(spa);
1336         hdr->b_state = arc_anon;
1337         hdr->b_arc_access = 0;
1338         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1339         buf->b_hdr = hdr;
1340         buf->b_data = NULL;
1341         buf->b_efunc = NULL;
1342         buf->b_private = NULL;
1343         buf->b_next = NULL;
1344         hdr->b_buf = buf;
1345         arc_get_data_buf(buf);
1346         hdr->b_datacnt = 1;
1347         hdr->b_flags = 0;
1348         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1349         (void) refcount_add(&hdr->b_refcnt, tag);
1350 
1351         return (buf);
1352 }
1353 
1354 static char *arc_onloan_tag = "onloan";
1355 
1356 /*
1357  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1358  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1359  * buffers must be returned to the arc before they can be used by the DMU or
1360  * freed.
1361  */
1362 arc_buf_t *
1363 arc_loan_buf(spa_t *spa, int size)
1364 {
1365         arc_buf_t *buf;
1366 
1367         buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1368 
1369         atomic_add_64(&arc_loaned_bytes, size);
1370         return (buf);
1371 }
1372 
1373 /*
1374  * Return a loaned arc buffer to the arc.
1375  */
1376 void
1377 arc_return_buf(arc_buf_t *buf, void *tag)
1378 {
1379         arc_buf_hdr_t *hdr = buf->b_hdr;
1380 
1381         ASSERT(buf->b_data != NULL);
1382         (void) refcount_add(&hdr->b_refcnt, tag);
1383         (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1384 
1385         atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1386 }
1387 
1388 /* Detach an arc_buf from a dbuf (tag) */
1389 void
1390 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1391 {
1392         arc_buf_hdr_t *hdr;
1393 
1394         ASSERT(buf->b_data != NULL);
1395         hdr = buf->b_hdr;
1396         (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1397         (void) refcount_remove(&hdr->b_refcnt, tag);
1398         buf->b_efunc = NULL;
1399         buf->b_private = NULL;
1400 
1401         atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1402 }
1403 
1404 static arc_buf_t *
1405 arc_buf_clone(arc_buf_t *from)
1406 {
1407         arc_buf_t *buf;
1408         arc_buf_hdr_t *hdr = from->b_hdr;
1409         uint64_t size = hdr->b_size;
1410 
1411         ASSERT(hdr->b_state != arc_anon);
1412 
1413         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1414         buf->b_hdr = hdr;
1415         buf->b_data = NULL;
1416         buf->b_efunc = NULL;
1417         buf->b_private = NULL;
1418         buf->b_next = hdr->b_buf;
1419         hdr->b_buf = buf;
1420         arc_get_data_buf(buf);
1421         bcopy(from->b_data, buf->b_data, size);
1422 
1423         /*
1424          * This buffer already exists in the arc so create a duplicate
1425          * copy for the caller.  If the buffer is associated with user data
1426          * then track the size and number of duplicates.  These stats will be
1427          * updated as duplicate buffers are created and destroyed.
1428          */
1429         if (hdr->b_type == ARC_BUFC_DATA) {
1430                 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1431                 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1432         }
1433         hdr->b_datacnt += 1;
1434         return (buf);
1435 }
1436 
1437 void
1438 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1439 {
1440         arc_buf_hdr_t *hdr;
1441         kmutex_t *hash_lock;
1442 
1443         /*
1444          * Check to see if this buffer is evicted.  Callers
1445          * must verify b_data != NULL to know if the add_ref
1446          * was successful.
1447          */
1448         mutex_enter(&buf->b_evict_lock);
1449         if (buf->b_data == NULL) {
1450                 mutex_exit(&buf->b_evict_lock);
1451                 return;
1452         }
1453         hash_lock = HDR_LOCK(buf->b_hdr);
1454         mutex_enter(hash_lock);
1455         hdr = buf->b_hdr;
1456         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1457         mutex_exit(&buf->b_evict_lock);
1458 
1459         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1460         add_reference(hdr, hash_lock, tag);
1461         DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1462         arc_access(hdr, hash_lock);
1463         mutex_exit(hash_lock);
1464         ARCSTAT_BUMP(arcstat_hits);
1465         ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1466             demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1467             data, metadata, hits);
1468 }
1469 
1470 /*
1471  * Free the arc data buffer.  If it is an l2arc write in progress,
1472  * the buffer is placed on l2arc_free_on_write to be freed later.
1473  */
1474 static void
1475 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1476 {
1477         arc_buf_hdr_t *hdr = buf->b_hdr;
1478 
1479         if (HDR_L2_WRITING(hdr)) {
1480                 l2arc_data_free_t *df;
1481                 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1482                 df->l2df_data = buf->b_data;
1483                 df->l2df_size = hdr->b_size;
1484                 df->l2df_func = free_func;
1485                 mutex_enter(&l2arc_free_on_write_mtx);
1486                 list_insert_head(l2arc_free_on_write, df);
1487                 mutex_exit(&l2arc_free_on_write_mtx);
1488                 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1489         } else {
1490                 free_func(buf->b_data, hdr->b_size);
1491         }
1492 }
1493 
1494 static void
1495 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1496 {
1497         arc_buf_t **bufp;
1498 
1499         /* free up data associated with the buf */
1500         if (buf->b_data) {
1501                 arc_state_t *state = buf->b_hdr->b_state;
1502                 uint64_t size = buf->b_hdr->b_size;
1503                 arc_buf_contents_t type = buf->b_hdr->b_type;
1504 
1505                 arc_cksum_verify(buf);
1506                 arc_buf_unwatch(buf);
1507 
1508                 if (!recycle) {
1509                         if (type == ARC_BUFC_METADATA) {
1510                                 arc_buf_data_free(buf, zio_buf_free);
1511                                 arc_space_return(size, ARC_SPACE_DATA);
1512                         } else {
1513                                 ASSERT(type == ARC_BUFC_DATA);
1514                                 arc_buf_data_free(buf, zio_data_buf_free);
1515                                 ARCSTAT_INCR(arcstat_data_size, -size);
1516                                 atomic_add_64(&arc_size, -size);
1517                         }
1518                 }
1519                 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1520                         uint64_t *cnt = &state->arcs_lsize[type];
1521 
1522                         ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
1523                         ASSERT(state != arc_anon);
1524 
1525                         ASSERT3U(*cnt, >=, size);
1526                         atomic_add_64(cnt, -size);
1527                 }
1528                 ASSERT3U(state->arcs_size, >=, size);
1529                 atomic_add_64(&state->arcs_size, -size);
1530                 buf->b_data = NULL;
1531 
1532                 /*
1533                  * If we're destroying a duplicate buffer make sure
1534                  * that the appropriate statistics are updated.
1535                  */
1536                 if (buf->b_hdr->b_datacnt > 1 &&
1537                     buf->b_hdr->b_type == ARC_BUFC_DATA) {
1538                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
1539                         ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
1540                 }
1541                 ASSERT(buf->b_hdr->b_datacnt > 0);
1542                 buf->b_hdr->b_datacnt -= 1;
1543         }
1544 
1545         /* only remove the buf if requested */
1546         if (!all)
1547                 return;
1548 
1549         /* remove the buf from the hdr list */
1550         for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1551                 continue;
1552         *bufp = buf->b_next;
1553         buf->b_next = NULL;
1554 
1555         ASSERT(buf->b_efunc == NULL);
1556 
1557         /* clean up the buf */
1558         buf->b_hdr = NULL;
1559         kmem_cache_free(buf_cache, buf);
1560 }
1561 
1562 static void
1563 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1564 {
1565         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1566         ASSERT3P(hdr->b_state, ==, arc_anon);
1567         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1568         l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1569 
1570         if (l2hdr != NULL) {
1571                 boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1572                 /*
1573                  * To prevent arc_free() and l2arc_evict() from
1574                  * attempting to free the same buffer at the same time,
1575                  * a FREE_IN_PROGRESS flag is given to arc_free() to
1576                  * give it priority.  l2arc_evict() can't destroy this
1577                  * header while we are waiting on l2arc_buflist_mtx.
1578                  *
1579                  * The hdr may be removed from l2ad_buflist before we
1580                  * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1581                  */
1582                 if (!buflist_held) {
1583                         mutex_enter(&l2arc_buflist_mtx);
1584                         l2hdr = hdr->b_l2hdr;
1585                 }
1586 
1587                 if (l2hdr != NULL) {
1588                         list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1589                         ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1590                         ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
1591                         kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1592                         if (hdr->b_state == arc_l2c_only)
1593                                 l2arc_hdr_stat_remove();
1594                         hdr->b_l2hdr = NULL;
1595                 }
1596 
1597                 if (!buflist_held)
1598                         mutex_exit(&l2arc_buflist_mtx);
1599         }
1600 
1601         if (!BUF_EMPTY(hdr)) {
1602                 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1603                 buf_discard_identity(hdr);
1604         }
1605         while (hdr->b_buf) {
1606                 arc_buf_t *buf = hdr->b_buf;
1607 
1608                 if (buf->b_efunc) {
1609                         mutex_enter(&arc_eviction_mtx);
1610                         mutex_enter(&buf->b_evict_lock);
1611                         ASSERT(buf->b_hdr != NULL);
1612                         arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1613                         hdr->b_buf = buf->b_next;
1614                         buf->b_hdr = &arc_eviction_hdr;
1615                         buf->b_next = arc_eviction_list;
1616                         arc_eviction_list = buf;
1617                         mutex_exit(&buf->b_evict_lock);
1618                         mutex_exit(&arc_eviction_mtx);
1619                 } else {
1620                         arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1621                 }
1622         }
1623         if (hdr->b_freeze_cksum != NULL) {
1624                 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1625                 hdr->b_freeze_cksum = NULL;
1626         }
1627         if (hdr->b_thawed) {
1628                 kmem_free(hdr->b_thawed, 1);
1629                 hdr->b_thawed = NULL;
1630         }
1631 
1632         ASSERT(!list_link_active(&hdr->b_arc_node));
1633         ASSERT3P(hdr->b_hash_next, ==, NULL);
1634         ASSERT3P(hdr->b_acb, ==, NULL);
1635         kmem_cache_free(hdr_cache, hdr);
1636 }
1637 
1638 void
1639 arc_buf_free(arc_buf_t *buf, void *tag)
1640 {
1641         arc_buf_hdr_t *hdr = buf->b_hdr;
1642         int hashed = hdr->b_state != arc_anon;
1643 
1644         ASSERT(buf->b_efunc == NULL);
1645         ASSERT(buf->b_data != NULL);
1646 
1647         if (hashed) {
1648                 kmutex_t *hash_lock = HDR_LOCK(hdr);
1649 
1650                 mutex_enter(hash_lock);
1651                 hdr = buf->b_hdr;
1652                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1653 
1654                 (void) remove_reference(hdr, hash_lock, tag);
1655                 if (hdr->b_datacnt > 1) {
1656                         arc_buf_destroy(buf, FALSE, TRUE);
1657                 } else {
1658                         ASSERT(buf == hdr->b_buf);
1659                         ASSERT(buf->b_efunc == NULL);
1660                         hdr->b_flags |= ARC_BUF_AVAILABLE;
1661                 }
1662                 mutex_exit(hash_lock);
1663         } else if (HDR_IO_IN_PROGRESS(hdr)) {
1664                 int destroy_hdr;
1665                 /*
1666                  * We are in the middle of an async write.  Don't destroy
1667                  * this buffer unless the write completes before we finish
1668                  * decrementing the reference count.
1669                  */
1670                 mutex_enter(&arc_eviction_mtx);
1671                 (void) remove_reference(hdr, NULL, tag);
1672                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1673                 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1674                 mutex_exit(&arc_eviction_mtx);
1675                 if (destroy_hdr)
1676                         arc_hdr_destroy(hdr);
1677         } else {
1678                 if (remove_reference(hdr, NULL, tag) > 0)
1679                         arc_buf_destroy(buf, FALSE, TRUE);
1680                 else
1681                         arc_hdr_destroy(hdr);
1682         }
1683 }
1684 
1685 boolean_t
1686 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1687 {
1688         arc_buf_hdr_t *hdr = buf->b_hdr;
1689         kmutex_t *hash_lock = HDR_LOCK(hdr);
1690         boolean_t no_callback = (buf->b_efunc == NULL);
1691 
1692         if (hdr->b_state == arc_anon) {
1693                 ASSERT(hdr->b_datacnt == 1);
1694                 arc_buf_free(buf, tag);
1695                 return (no_callback);
1696         }
1697 
1698         mutex_enter(hash_lock);
1699         hdr = buf->b_hdr;
1700         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1701         ASSERT(hdr->b_state != arc_anon);
1702         ASSERT(buf->b_data != NULL);
1703 
1704         (void) remove_reference(hdr, hash_lock, tag);
1705         if (hdr->b_datacnt > 1) {
1706                 if (no_callback)
1707                         arc_buf_destroy(buf, FALSE, TRUE);
1708         } else if (no_callback) {
1709                 ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1710                 ASSERT(buf->b_efunc == NULL);
1711                 hdr->b_flags |= ARC_BUF_AVAILABLE;
1712         }
1713         ASSERT(no_callback || hdr->b_datacnt > 1 ||
1714             refcount_is_zero(&hdr->b_refcnt));
1715         mutex_exit(hash_lock);
1716         return (no_callback);
1717 }
1718 
1719 int
1720 arc_buf_size(arc_buf_t *buf)
1721 {
1722         return (buf->b_hdr->b_size);
1723 }
1724 
1725 /*
1726  * Called from the DMU to determine if the current buffer should be
1727  * evicted. In order to ensure proper locking, the eviction must be initiated
1728  * from the DMU. Return true if the buffer is associated with user data and
1729  * duplicate buffers still exist.
1730  */
1731 boolean_t
1732 arc_buf_eviction_needed(arc_buf_t *buf)
1733 {
1734         arc_buf_hdr_t *hdr;
1735         boolean_t evict_needed = B_FALSE;
1736 
1737         if (zfs_disable_dup_eviction)
1738                 return (B_FALSE);
1739 
1740         mutex_enter(&buf->b_evict_lock);
1741         hdr = buf->b_hdr;
1742         if (hdr == NULL) {
1743                 /*
1744                  * We are in arc_do_user_evicts(); let that function
1745                  * perform the eviction.
1746                  */
1747                 ASSERT(buf->b_data == NULL);
1748                 mutex_exit(&buf->b_evict_lock);
1749                 return (B_FALSE);
1750         } else if (buf->b_data == NULL) {
1751                 /*
1752                  * We have already been added to the arc eviction list;
1753                  * recommend eviction.
1754                  */
1755                 ASSERT3P(hdr, ==, &arc_eviction_hdr);
1756                 mutex_exit(&buf->b_evict_lock);
1757                 return (B_TRUE);
1758         }
1759 
1760         if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
1761                 evict_needed = B_TRUE;
1762 
1763         mutex_exit(&buf->b_evict_lock);
1764         return (evict_needed);
1765 }
1766 
1767 /*
1768  * Evict buffers from list until we've removed the specified number of
1769  * bytes.  Move the removed buffers to the appropriate evict state.
1770  * If the recycle flag is set, then attempt to "recycle" a buffer:
1771  * - look for a buffer to evict that is `bytes' long.
1772  * - return the data block from this buffer rather than freeing it.
1773  * This flag is used by callers that are trying to make space for a
1774  * new buffer in a full arc cache.
1775  *
1776  * This function makes a "best effort".  It skips over any buffers
1777  * it can't get a hash_lock on, and so may not catch all candidates.
1778  * It may also return without evicting as much space as requested.
1779  */
1780 static void *
1781 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1782     arc_buf_contents_t type)
1783 {
1784         arc_state_t *evicted_state;
1785         uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1786         arc_buf_hdr_t *ab, *ab_prev = NULL;
1787         list_t *list = &state->arcs_list[type];
1788         kmutex_t *hash_lock;
1789         boolean_t have_lock;
1790         void *stolen = NULL;
1791         arc_buf_hdr_t marker = { 0 };
1792         int count = 0;
1793 
1794         ASSERT(state == arc_mru || state == arc_mfu);
1795 
1796         evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1797 
1798         mutex_enter(&state->arcs_mtx);
1799         mutex_enter(&evicted_state->arcs_mtx);
1800 
1801         for (ab = list_tail(list); ab; ab = ab_prev) {
1802                 ab_prev = list_prev(list, ab);
1803                 /* prefetch buffers have a minimum lifespan */
1804                 if (HDR_IO_IN_PROGRESS(ab) ||
1805                     (spa && ab->b_spa != spa) ||
1806                     (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1807                     ddi_get_lbolt() - ab->b_arc_access <
1808                     arc_min_prefetch_lifespan)) {
1809                         skipped++;
1810                         continue;
1811                 }
1812                 /* "lookahead" for better eviction candidate */
1813                 if (recycle && ab->b_size != bytes &&
1814                     ab_prev && ab_prev->b_size == bytes)
1815                         continue;
1816 
1817                 /* ignore markers */
1818                 if (ab->b_spa == 0)
1819                         continue;
1820 
1821                 /*
1822                  * It may take a long time to evict all the bufs requested.
1823                  * To avoid blocking all arc activity, periodically drop
1824                  * the arcs_mtx and give other threads a chance to run
1825                  * before reacquiring the lock.
1826                  *
1827                  * If we are looking for a buffer to recycle, we are in
1828                  * the hot code path, so don't sleep.
1829                  */
1830                 if (!recycle && count++ > arc_evict_iterations) {
1831                         list_insert_after(list, ab, &marker);
1832                         mutex_exit(&evicted_state->arcs_mtx);
1833                         mutex_exit(&state->arcs_mtx);
1834                         kpreempt(KPREEMPT_SYNC);
1835                         mutex_enter(&state->arcs_mtx);
1836                         mutex_enter(&evicted_state->arcs_mtx);
1837                         ab_prev = list_prev(list, &marker);
1838                         list_remove(list, &marker);
1839                         count = 0;
1840                         continue;
1841                 }
1842 
1843                 hash_lock = HDR_LOCK(ab);
1844                 have_lock = MUTEX_HELD(hash_lock);
1845                 if (have_lock || mutex_tryenter(hash_lock)) {
1846                         ASSERT0(refcount_count(&ab->b_refcnt));
1847                         ASSERT(ab->b_datacnt > 0);
1848                         while (ab->b_buf) {
1849                                 arc_buf_t *buf = ab->b_buf;
1850                                 if (!mutex_tryenter(&buf->b_evict_lock)) {
1851                                         missed += 1;
1852                                         break;
1853                                 }
1854                                 if (buf->b_data) {
1855                                         bytes_evicted += ab->b_size;
1856                                         if (recycle && ab->b_type == type &&
1857                                             ab->b_size == bytes &&
1858                                             !HDR_L2_WRITING(ab)) {
1859                                                 stolen = buf->b_data;
1860                                                 recycle = FALSE;
1861                                         }
1862                                 }
1863                                 if (buf->b_efunc) {
1864                                         mutex_enter(&arc_eviction_mtx);
1865                                         arc_buf_destroy(buf,
1866                                             buf->b_data == stolen, FALSE);
1867                                         ab->b_buf = buf->b_next;
1868                                         buf->b_hdr = &arc_eviction_hdr;
1869                                         buf->b_next = arc_eviction_list;
1870                                         arc_eviction_list = buf;
1871                                         mutex_exit(&arc_eviction_mtx);
1872                                         mutex_exit(&buf->b_evict_lock);
1873                                 } else {
1874                                         mutex_exit(&buf->b_evict_lock);
1875                                         arc_buf_destroy(buf,
1876                                             buf->b_data == stolen, TRUE);
1877                                 }
1878                         }
1879 
1880                         if (ab->b_l2hdr) {
1881                                 ARCSTAT_INCR(arcstat_evict_l2_cached,
1882                                     ab->b_size);
1883                         } else {
1884                                 if (l2arc_write_eligible(ab->b_spa, ab)) {
1885                                         ARCSTAT_INCR(arcstat_evict_l2_eligible,
1886                                             ab->b_size);
1887                                 } else {
1888                                         ARCSTAT_INCR(
1889                                             arcstat_evict_l2_ineligible,
1890                                             ab->b_size);
1891                                 }
1892                         }
1893 
1894                         if (ab->b_datacnt == 0) {
1895                                 arc_change_state(evicted_state, ab, hash_lock);
1896                                 ASSERT(HDR_IN_HASH_TABLE(ab));
1897                                 ab->b_flags |= ARC_IN_HASH_TABLE;
1898                                 ab->b_flags &= ~ARC_BUF_AVAILABLE;
1899                                 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
1900                         }
1901                         if (!have_lock)
1902                                 mutex_exit(hash_lock);
1903                         if (bytes >= 0 && bytes_evicted >= bytes)
1904                                 break;
1905                 } else {
1906                         missed += 1;
1907                 }
1908         }
1909 
1910         mutex_exit(&evicted_state->arcs_mtx);
1911         mutex_exit(&state->arcs_mtx);
1912 
1913         if (bytes_evicted < bytes)
1914                 dprintf("only evicted %lld bytes from %x",
1915                     (longlong_t)bytes_evicted, state);
1916 
1917         if (skipped)
1918                 ARCSTAT_INCR(arcstat_evict_skip, skipped);
1919 
1920         if (missed)
1921                 ARCSTAT_INCR(arcstat_mutex_miss, missed);
1922 
1923         /*
1924          * Note: we have just evicted some data into the ghost state,
1925          * potentially putting the ghost size over the desired size.  Rather
1926          * that evicting from the ghost list in this hot code path, leave
1927          * this chore to the arc_reclaim_thread().
1928          */
1929 
1930         return (stolen);
1931 }
1932 
1933 /*
1934  * Remove buffers from list until we've removed the specified number of
1935  * bytes.  Destroy the buffers that are removed.
1936  */
1937 static void
1938 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1939 {
1940         arc_buf_hdr_t *ab, *ab_prev;
1941         arc_buf_hdr_t marker = { 0 };
1942         list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1943         kmutex_t *hash_lock;
1944         uint64_t bytes_deleted = 0;
1945         uint64_t bufs_skipped = 0;
1946         int count = 0;
1947 
1948         ASSERT(GHOST_STATE(state));
1949 top:
1950         mutex_enter(&state->arcs_mtx);
1951         for (ab = list_tail(list); ab; ab = ab_prev) {
1952                 ab_prev = list_prev(list, ab);
1953                 if (ab->b_type > ARC_BUFC_NUMTYPES)
1954                         panic("invalid ab=%p", (void *)ab);
1955                 if (spa && ab->b_spa != spa)
1956                         continue;
1957 
1958                 /* ignore markers */
1959                 if (ab->b_spa == 0)
1960                         continue;
1961 
1962                 hash_lock = HDR_LOCK(ab);
1963                 /* caller may be trying to modify this buffer, skip it */
1964                 if (MUTEX_HELD(hash_lock))
1965                         continue;
1966 
1967                 /*
1968                  * It may take a long time to evict all the bufs requested.
1969                  * To avoid blocking all arc activity, periodically drop
1970                  * the arcs_mtx and give other threads a chance to run
1971                  * before reacquiring the lock.
1972                  */
1973                 if (count++ > arc_evict_iterations) {
1974                         list_insert_after(list, ab, &marker);
1975                         mutex_exit(&state->arcs_mtx);
1976                         kpreempt(KPREEMPT_SYNC);
1977                         mutex_enter(&state->arcs_mtx);
1978                         ab_prev = list_prev(list, &marker);
1979                         list_remove(list, &marker);
1980                         count = 0;
1981                         continue;
1982                 }
1983                 if (mutex_tryenter(hash_lock)) {
1984                         ASSERT(!HDR_IO_IN_PROGRESS(ab));
1985                         ASSERT(ab->b_buf == NULL);
1986                         ARCSTAT_BUMP(arcstat_deleted);
1987                         bytes_deleted += ab->b_size;
1988 
1989                         if (ab->b_l2hdr != NULL) {
1990                                 /*
1991                                  * This buffer is cached on the 2nd Level ARC;
1992                                  * don't destroy the header.
1993                                  */
1994                                 arc_change_state(arc_l2c_only, ab, hash_lock);
1995                                 mutex_exit(hash_lock);
1996                         } else {
1997                                 arc_change_state(arc_anon, ab, hash_lock);
1998                                 mutex_exit(hash_lock);
1999                                 arc_hdr_destroy(ab);
2000                         }
2001 
2002                         DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
2003                         if (bytes >= 0 && bytes_deleted >= bytes)
2004                                 break;
2005                 } else if (bytes < 0) {
2006                         /*
2007                          * Insert a list marker and then wait for the
2008                          * hash lock to become available. Once its
2009                          * available, restart from where we left off.
2010                          */
2011                         list_insert_after(list, ab, &marker);
2012                         mutex_exit(&state->arcs_mtx);
2013                         mutex_enter(hash_lock);
2014                         mutex_exit(hash_lock);
2015                         mutex_enter(&state->arcs_mtx);
2016                         ab_prev = list_prev(list, &marker);
2017                         list_remove(list, &marker);
2018                 } else {
2019                         bufs_skipped += 1;
2020                 }
2021 
2022         }
2023         mutex_exit(&state->arcs_mtx);
2024 
2025         if (list == &state->arcs_list[ARC_BUFC_DATA] &&
2026             (bytes < 0 || bytes_deleted < bytes)) {
2027                 list = &state->arcs_list[ARC_BUFC_METADATA];
2028                 goto top;
2029         }
2030 
2031         if (bufs_skipped) {
2032                 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
2033                 ASSERT(bytes >= 0);
2034         }
2035 
2036         if (bytes_deleted < bytes)
2037                 dprintf("only deleted %lld bytes from %p",
2038                     (longlong_t)bytes_deleted, state);
2039 }
2040 
2041 static void
2042 arc_adjust(void)
2043 {
2044         int64_t adjustment, delta;
2045 
2046         /*
2047          * Adjust MRU size
2048          */
2049 
2050         adjustment = MIN((int64_t)(arc_size - arc_c),
2051             (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
2052             arc_p));
2053 
2054         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
2055                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
2056                 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
2057                 adjustment -= delta;
2058         }
2059 
2060         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2061                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
2062                 (void) arc_evict(arc_mru, NULL, delta, FALSE,
2063                     ARC_BUFC_METADATA);
2064         }
2065 
2066         /*
2067          * Adjust MFU size
2068          */
2069 
2070         adjustment = arc_size - arc_c;
2071 
2072         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
2073                 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
2074                 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
2075                 adjustment -= delta;
2076         }
2077 
2078         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2079                 int64_t delta = MIN(adjustment,
2080                     arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
2081                 (void) arc_evict(arc_mfu, NULL, delta, FALSE,
2082                     ARC_BUFC_METADATA);
2083         }
2084 
2085         /*
2086          * Adjust ghost lists
2087          */
2088 
2089         adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2090 
2091         if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
2092                 delta = MIN(arc_mru_ghost->arcs_size, adjustment);
2093                 arc_evict_ghost(arc_mru_ghost, NULL, delta);
2094         }
2095 
2096         adjustment =
2097             arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2098 
2099         if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2100                 delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2101                 arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2102         }
2103 }
2104 
2105 static void
2106 arc_do_user_evicts(void)
2107 {
2108         mutex_enter(&arc_eviction_mtx);
2109         while (arc_eviction_list != NULL) {
2110                 arc_buf_t *buf = arc_eviction_list;
2111                 arc_eviction_list = buf->b_next;
2112                 mutex_enter(&buf->b_evict_lock);
2113                 buf->b_hdr = NULL;
2114                 mutex_exit(&buf->b_evict_lock);
2115                 mutex_exit(&arc_eviction_mtx);
2116 
2117                 if (buf->b_efunc != NULL)
2118                         VERIFY(buf->b_efunc(buf) == 0);
2119 
2120                 buf->b_efunc = NULL;
2121                 buf->b_private = NULL;
2122                 kmem_cache_free(buf_cache, buf);
2123                 mutex_enter(&arc_eviction_mtx);
2124         }
2125         mutex_exit(&arc_eviction_mtx);
2126 }
2127 
2128 /*
2129  * Flush all *evictable* data from the cache for the given spa.
2130  * NOTE: this will not touch "active" (i.e. referenced) data.
2131  */
2132 void
2133 arc_flush(spa_t *spa)
2134 {
2135         uint64_t guid = 0;
2136 
2137         if (spa)
2138                 guid = spa_load_guid(spa);
2139 
2140         while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
2141                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2142                 if (spa)
2143                         break;
2144         }
2145         while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
2146                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2147                 if (spa)
2148                         break;
2149         }
2150         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
2151                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2152                 if (spa)
2153                         break;
2154         }
2155         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
2156                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2157                 if (spa)
2158                         break;
2159         }
2160 
2161         arc_evict_ghost(arc_mru_ghost, guid, -1);
2162         arc_evict_ghost(arc_mfu_ghost, guid, -1);
2163 
2164         mutex_enter(&arc_reclaim_thr_lock);
2165         arc_do_user_evicts();
2166         mutex_exit(&arc_reclaim_thr_lock);
2167         ASSERT(spa || arc_eviction_list == NULL);
2168 }
2169 
2170 void
2171 arc_shrink(void)
2172 {
2173         if (arc_c > arc_c_min) {
2174                 uint64_t to_free;
2175 
2176 #ifdef _KERNEL
2177                 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
2178 #else
2179                 to_free = arc_c >> arc_shrink_shift;
2180 #endif
2181                 if (arc_c > arc_c_min + to_free)
2182                         atomic_add_64(&arc_c, -to_free);
2183                 else
2184                         arc_c = arc_c_min;
2185 
2186                 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
2187                 if (arc_c > arc_size)
2188                         arc_c = MAX(arc_size, arc_c_min);
2189                 if (arc_p > arc_c)
2190                         arc_p = (arc_c >> 1);
2191                 ASSERT(arc_c >= arc_c_min);
2192                 ASSERT((int64_t)arc_p >= 0);
2193         }
2194 
2195         if (arc_size > arc_c)
2196                 arc_adjust();
2197 }
2198 
2199 /*
2200  * Determine if the system is under memory pressure and is asking
2201  * to reclaim memory. A return value of 1 indicates that the system
2202  * is under memory pressure and that the arc should adjust accordingly.
2203  */
2204 static int
2205 arc_reclaim_needed(void)
2206 {
2207         uint64_t extra;
2208 
2209 #ifdef _KERNEL
2210 
2211         if (needfree)
2212                 return (1);
2213 
2214         /*
2215          * take 'desfree' extra pages, so we reclaim sooner, rather than later
2216          */
2217         extra = desfree;
2218 
2219         /*
2220          * check that we're out of range of the pageout scanner.  It starts to
2221          * schedule paging if freemem is less than lotsfree and needfree.
2222          * lotsfree is the high-water mark for pageout, and needfree is the
2223          * number of needed free pages.  We add extra pages here to make sure
2224          * the scanner doesn't start up while we're freeing memory.
2225          */
2226         if (freemem < lotsfree + needfree + extra)
2227                 return (1);
2228 
2229         /*
2230          * check to make sure that swapfs has enough space so that anon
2231          * reservations can still succeed. anon_resvmem() checks that the
2232          * availrmem is greater than swapfs_minfree, and the number of reserved
2233          * swap pages.  We also add a bit of extra here just to prevent
2234          * circumstances from getting really dire.
2235          */
2236         if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2237                 return (1);
2238 
2239         /*
2240          * Check that we have enough availrmem that memory locking (e.g., via
2241          * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
2242          * stores the number of pages that cannot be locked; when availrmem
2243          * drops below pages_pp_maximum, page locking mechanisms such as
2244          * page_pp_lock() will fail.)
2245          */
2246         if (availrmem <= pages_pp_maximum)
2247                 return (1);
2248 
2249 #if defined(__i386)
2250         /*
2251          * If we're on an i386 platform, it's possible that we'll exhaust the
2252          * kernel heap space before we ever run out of available physical
2253          * memory.  Most checks of the size of the heap_area compare against
2254          * tune.t_minarmem, which is the minimum available real memory that we
2255          * can have in the system.  However, this is generally fixed at 25 pages
2256          * which is so low that it's useless.  In this comparison, we seek to
2257          * calculate the total heap-size, and reclaim if more than 3/4ths of the
2258          * heap is allocated.  (Or, in the calculation, if less than 1/4th is
2259          * free)
2260          */
2261         if (vmem_size(heap_arena, VMEM_FREE) <
2262             (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
2263                 return (1);
2264 #endif
2265 
2266         /*
2267          * If zio data pages are being allocated out of a separate heap segment,
2268          * then enforce that the size of available vmem for this arena remains
2269          * above about 1/16th free.
2270          *
2271          * Note: The 1/16th arena free requirement was put in place
2272          * to aggressively evict memory from the arc in order to avoid
2273          * memory fragmentation issues.
2274          */
2275         if (zio_arena != NULL &&
2276             vmem_size(zio_arena, VMEM_FREE) <
2277             (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
2278                 return (1);
2279 #else
2280         if (spa_get_random(100) == 0)
2281                 return (1);
2282 #endif
2283         return (0);
2284 }
2285 
2286 static void
2287 arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2288 {
2289         size_t                  i;
2290         kmem_cache_t            *prev_cache = NULL;
2291         kmem_cache_t            *prev_data_cache = NULL;
2292         extern kmem_cache_t     *zio_buf_cache[];
2293         extern kmem_cache_t     *zio_data_buf_cache[];
2294 
2295 #ifdef _KERNEL
2296         if (arc_meta_used >= arc_meta_limit) {
2297                 /*
2298                  * We are exceeding our meta-data cache limit.
2299                  * Purge some DNLC entries to release holds on meta-data.
2300                  */
2301                 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
2302         }
2303 #if defined(__i386)
2304         /*
2305          * Reclaim unused memory from all kmem caches.
2306          */
2307         kmem_reap();
2308 #endif
2309 #endif
2310 
2311         /*
2312          * An aggressive reclamation will shrink the cache size as well as
2313          * reap free buffers from the arc kmem caches.
2314          */
2315         if (strat == ARC_RECLAIM_AGGR)
2316                 arc_shrink();
2317 
2318         for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2319                 if (zio_buf_cache[i] != prev_cache) {
2320                         prev_cache = zio_buf_cache[i];
2321                         kmem_cache_reap_now(zio_buf_cache[i]);
2322                 }
2323                 if (zio_data_buf_cache[i] != prev_data_cache) {
2324                         prev_data_cache = zio_data_buf_cache[i];
2325                         kmem_cache_reap_now(zio_data_buf_cache[i]);
2326                 }
2327         }
2328         kmem_cache_reap_now(buf_cache);
2329         kmem_cache_reap_now(hdr_cache);
2330 
2331         /*
2332          * Ask the vmem areana to reclaim unused memory from its
2333          * quantum caches.
2334          */
2335         if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
2336                 vmem_qcache_reap(zio_arena);
2337 }
2338 
2339 static void
2340 arc_reclaim_thread(void)
2341 {
2342         clock_t                 growtime = 0;
2343         arc_reclaim_strategy_t  last_reclaim = ARC_RECLAIM_CONS;
2344         callb_cpr_t             cpr;
2345 
2346         CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2347 
2348         mutex_enter(&arc_reclaim_thr_lock);
2349         while (arc_thread_exit == 0) {
2350                 if (arc_reclaim_needed()) {
2351 
2352                         if (arc_no_grow) {
2353                                 if (last_reclaim == ARC_RECLAIM_CONS) {
2354                                         last_reclaim = ARC_RECLAIM_AGGR;
2355                                 } else {
2356                                         last_reclaim = ARC_RECLAIM_CONS;
2357                                 }
2358                         } else {
2359                                 arc_no_grow = TRUE;
2360                                 last_reclaim = ARC_RECLAIM_AGGR;
2361                                 membar_producer();
2362                         }
2363 
2364                         /* reset the growth delay for every reclaim */
2365                         growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2366 
2367                         arc_kmem_reap_now(last_reclaim);
2368                         arc_warm = B_TRUE;
2369 
2370                 } else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
2371                         arc_no_grow = FALSE;
2372                 }
2373 
2374                 arc_adjust();
2375 
2376                 if (arc_eviction_list != NULL)
2377                         arc_do_user_evicts();
2378 
2379                 /* block until needed, or one second, whichever is shorter */
2380                 CALLB_CPR_SAFE_BEGIN(&cpr);
2381                 (void) cv_timedwait(&arc_reclaim_thr_cv,
2382                     &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2383                 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2384         }
2385 
2386         arc_thread_exit = 0;
2387         cv_broadcast(&arc_reclaim_thr_cv);
2388         CALLB_CPR_EXIT(&cpr);               /* drops arc_reclaim_thr_lock */
2389         thread_exit();
2390 }
2391 
2392 /*
2393  * Adapt arc info given the number of bytes we are trying to add and
2394  * the state that we are comming from.  This function is only called
2395  * when we are adding new content to the cache.
2396  */
2397 static void
2398 arc_adapt(int bytes, arc_state_t *state)
2399 {
2400         int mult;
2401         uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2402 
2403         if (state == arc_l2c_only)
2404                 return;
2405 
2406         ASSERT(bytes > 0);
2407         /*
2408          * Adapt the target size of the MRU list:
2409          *      - if we just hit in the MRU ghost list, then increase
2410          *        the target size of the MRU list.
2411          *      - if we just hit in the MFU ghost list, then increase
2412          *        the target size of the MFU list by decreasing the
2413          *        target size of the MRU list.
2414          */
2415         if (state == arc_mru_ghost) {
2416                 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
2417                     1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
2418                 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
2419 
2420                 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
2421         } else if (state == arc_mfu_ghost) {
2422                 uint64_t delta;
2423 
2424                 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
2425                     1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
2426                 mult = MIN(mult, 10);
2427 
2428                 delta = MIN(bytes * mult, arc_p);
2429                 arc_p = MAX(arc_p_min, arc_p - delta);
2430         }
2431         ASSERT((int64_t)arc_p >= 0);
2432 
2433         if (arc_reclaim_needed()) {
2434                 cv_signal(&arc_reclaim_thr_cv);
2435                 return;
2436         }
2437 
2438         if (arc_no_grow)
2439                 return;
2440 
2441         if (arc_c >= arc_c_max)
2442                 return;
2443 
2444         /*
2445          * If we're within (2 * maxblocksize) bytes of the target
2446          * cache size, increment the target cache size
2447          */
2448         if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
2449                 atomic_add_64(&arc_c, (int64_t)bytes);
2450                 if (arc_c > arc_c_max)
2451                         arc_c = arc_c_max;
2452                 else if (state == arc_anon)
2453                         atomic_add_64(&arc_p, (int64_t)bytes);
2454                 if (arc_p > arc_c)
2455                         arc_p = arc_c;
2456         }
2457         ASSERT((int64_t)arc_p >= 0);
2458 }
2459 
2460 /*
2461  * Check if the cache has reached its limits and eviction is required
2462  * prior to insert.
2463  */
2464 static int
2465 arc_evict_needed(arc_buf_contents_t type)
2466 {
2467         if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2468                 return (1);
2469 
2470         if (arc_reclaim_needed())
2471                 return (1);
2472 
2473         return (arc_size > arc_c);
2474 }
2475 
2476 /*
2477  * The buffer, supplied as the first argument, needs a data block.
2478  * So, if we are at cache max, determine which cache should be victimized.
2479  * We have the following cases:
2480  *
2481  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2482  * In this situation if we're out of space, but the resident size of the MFU is
2483  * under the limit, victimize the MFU cache to satisfy this insertion request.
2484  *
2485  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2486  * Here, we've used up all of the available space for the MRU, so we need to
2487  * evict from our own cache instead.  Evict from the set of resident MRU
2488  * entries.
2489  *
2490  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2491  * c minus p represents the MFU space in the cache, since p is the size of the
2492  * cache that is dedicated to the MRU.  In this situation there's still space on
2493  * the MFU side, so the MRU side needs to be victimized.
2494  *
2495  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2496  * MFU's resident set is consuming more space than it has been allotted.  In
2497  * this situation, we must victimize our own cache, the MFU, for this insertion.
2498  */
2499 static void
2500 arc_get_data_buf(arc_buf_t *buf)
2501 {
2502         arc_state_t             *state = buf->b_hdr->b_state;
2503         uint64_t                size = buf->b_hdr->b_size;
2504         arc_buf_contents_t      type = buf->b_hdr->b_type;
2505 
2506         arc_adapt(size, state);
2507 
2508         /*
2509          * We have not yet reached cache maximum size,
2510          * just allocate a new buffer.
2511          */
2512         if (!arc_evict_needed(type)) {
2513                 if (type == ARC_BUFC_METADATA) {
2514                         buf->b_data = zio_buf_alloc(size);
2515                         arc_space_consume(size, ARC_SPACE_DATA);
2516                 } else {
2517                         ASSERT(type == ARC_BUFC_DATA);
2518                         buf->b_data = zio_data_buf_alloc(size);
2519                         ARCSTAT_INCR(arcstat_data_size, size);
2520                         atomic_add_64(&arc_size, size);
2521                 }
2522                 goto out;
2523         }
2524 
2525         /*
2526          * If we are prefetching from the mfu ghost list, this buffer
2527          * will end up on the mru list; so steal space from there.
2528          */
2529         if (state == arc_mfu_ghost)
2530                 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
2531         else if (state == arc_mru_ghost)
2532                 state = arc_mru;
2533 
2534         if (state == arc_mru || state == arc_anon) {
2535                 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
2536                 state = (arc_mfu->arcs_lsize[type] >= size &&
2537                     arc_p > mru_used) ? arc_mfu : arc_mru;
2538         } else {
2539                 /* MFU cases */
2540                 uint64_t mfu_space = arc_c - arc_p;
2541                 state =  (arc_mru->arcs_lsize[type] >= size &&
2542                     mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2543         }
2544         if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2545                 if (type == ARC_BUFC_METADATA) {
2546                         buf->b_data = zio_buf_alloc(size);
2547                         arc_space_consume(size, ARC_SPACE_DATA);
2548                 } else {
2549                         ASSERT(type == ARC_BUFC_DATA);
2550                         buf->b_data = zio_data_buf_alloc(size);
2551                         ARCSTAT_INCR(arcstat_data_size, size);
2552                         atomic_add_64(&arc_size, size);
2553                 }
2554                 ARCSTAT_BUMP(arcstat_recycle_miss);
2555         }
2556         ASSERT(buf->b_data != NULL);
2557 out:
2558         /*
2559          * Update the state size.  Note that ghost states have a
2560          * "ghost size" and so don't need to be updated.
2561          */
2562         if (!GHOST_STATE(buf->b_hdr->b_state)) {
2563                 arc_buf_hdr_t *hdr = buf->b_hdr;
2564 
2565                 atomic_add_64(&hdr->b_state->arcs_size, size);
2566                 if (list_link_active(&hdr->b_arc_node)) {
2567                         ASSERT(refcount_is_zero(&hdr->b_refcnt));
2568                         atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2569                 }
2570                 /*
2571                  * If we are growing the cache, and we are adding anonymous
2572                  * data, and we have outgrown arc_p, update arc_p
2573                  */
2574                 if (arc_size < arc_c && hdr->b_state == arc_anon &&
2575                     arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
2576                         arc_p = MIN(arc_c, arc_p + size);
2577         }
2578 }
2579 
2580 /*
2581  * This routine is called whenever a buffer is accessed.
2582  * NOTE: the hash lock is dropped in this function.
2583  */
2584 static void
2585 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2586 {
2587         clock_t now;
2588 
2589         ASSERT(MUTEX_HELD(hash_lock));
2590 
2591         if (buf->b_state == arc_anon) {
2592                 /*
2593                  * This buffer is not in the cache, and does not
2594                  * appear in our "ghost" list.  Add the new buffer
2595                  * to the MRU state.
2596                  */
2597 
2598                 ASSERT(buf->b_arc_access == 0);
2599                 buf->b_arc_access = ddi_get_lbolt();
2600                 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2601                 arc_change_state(arc_mru, buf, hash_lock);
2602 
2603         } else if (buf->b_state == arc_mru) {
2604                 now = ddi_get_lbolt();
2605 
2606                 /*
2607                  * If this buffer is here because of a prefetch, then either:
2608                  * - clear the flag if this is a "referencing" read
2609                  *   (any subsequent access will bump this into the MFU state).
2610                  * or
2611                  * - move the buffer to the head of the list if this is
2612                  *   another prefetch (to make it less likely to be evicted).
2613                  */
2614                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2615                         if (refcount_count(&buf->b_refcnt) == 0) {
2616                                 ASSERT(list_link_active(&buf->b_arc_node));
2617                         } else {
2618                                 buf->b_flags &= ~ARC_PREFETCH;
2619                                 ARCSTAT_BUMP(arcstat_mru_hits);
2620                         }
2621                         buf->b_arc_access = now;
2622                         return;
2623                 }
2624 
2625                 /*
2626                  * This buffer has been "accessed" only once so far,
2627                  * but it is still in the cache. Move it to the MFU
2628                  * state.
2629                  */
2630                 if (now > buf->b_arc_access + ARC_MINTIME) {
2631                         /*
2632                          * More than 125ms have passed since we
2633                          * instantiated this buffer.  Move it to the
2634                          * most frequently used state.
2635                          */
2636                         buf->b_arc_access = now;
2637                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2638                         arc_change_state(arc_mfu, buf, hash_lock);
2639                 }
2640                 ARCSTAT_BUMP(arcstat_mru_hits);
2641         } else if (buf->b_state == arc_mru_ghost) {
2642                 arc_state_t     *new_state;
2643                 /*
2644                  * This buffer has been "accessed" recently, but
2645                  * was evicted from the cache.  Move it to the
2646                  * MFU state.
2647                  */
2648 
2649                 if (buf->b_flags & ARC_PREFETCH) {
2650                         new_state = arc_mru;
2651                         if (refcount_count(&buf->b_refcnt) > 0)
2652                                 buf->b_flags &= ~ARC_PREFETCH;
2653                         DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2654                 } else {
2655                         new_state = arc_mfu;
2656                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2657                 }
2658 
2659                 buf->b_arc_access = ddi_get_lbolt();
2660                 arc_change_state(new_state, buf, hash_lock);
2661 
2662                 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
2663         } else if (buf->b_state == arc_mfu) {
2664                 /*
2665                  * This buffer has been accessed more than once and is
2666                  * still in the cache.  Keep it in the MFU state.
2667                  *
2668                  * NOTE: an add_reference() that occurred when we did
2669                  * the arc_read() will have kicked this off the list.
2670                  * If it was a prefetch, we will explicitly move it to
2671                  * the head of the list now.
2672                  */
2673                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2674                         ASSERT(refcount_count(&buf->b_refcnt) == 0);
2675                         ASSERT(list_link_active(&buf->b_arc_node));
2676                 }
2677                 ARCSTAT_BUMP(arcstat_mfu_hits);
2678                 buf->b_arc_access = ddi_get_lbolt();
2679         } else if (buf->b_state == arc_mfu_ghost) {
2680                 arc_state_t     *new_state = arc_mfu;
2681                 /*
2682                  * This buffer has been accessed more than once but has
2683                  * been evicted from the cache.  Move it back to the
2684                  * MFU state.
2685                  */
2686 
2687                 if (buf->b_flags & ARC_PREFETCH) {
2688                         /*
2689                          * This is a prefetch access...
2690                          * move this block back to the MRU state.
2691                          */
2692                         ASSERT0(refcount_count(&buf->b_refcnt));
2693                         new_state = arc_mru;
2694                 }
2695 
2696                 buf->b_arc_access = ddi_get_lbolt();
2697                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2698                 arc_change_state(new_state, buf, hash_lock);
2699 
2700                 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2701         } else if (buf->b_state == arc_l2c_only) {
2702                 /*
2703                  * This buffer is on the 2nd Level ARC.
2704                  */
2705 
2706                 buf->b_arc_access = ddi_get_lbolt();
2707                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2708                 arc_change_state(arc_mfu, buf, hash_lock);
2709         } else {
2710                 ASSERT(!"invalid arc state");
2711         }
2712 }
2713 
2714 /* a generic arc_done_func_t which you can use */
2715 /* ARGSUSED */
2716 void
2717 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2718 {
2719         if (zio == NULL || zio->io_error == 0)
2720                 bcopy(buf->b_data, arg, buf->b_hdr->b_size);
2721         VERIFY(arc_buf_remove_ref(buf, arg));
2722 }
2723 
2724 /* a generic arc_done_func_t */
2725 void
2726 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2727 {
2728         arc_buf_t **bufp = arg;
2729         if (zio && zio->io_error) {
2730                 VERIFY(arc_buf_remove_ref(buf, arg));
2731                 *bufp = NULL;
2732         } else {
2733                 *bufp = buf;
2734                 ASSERT(buf->b_data);
2735         }
2736 }
2737 
2738 static void
2739 arc_read_done(zio_t *zio)
2740 {
2741         arc_buf_hdr_t   *hdr, *found;
2742         arc_buf_t       *buf;
2743         arc_buf_t       *abuf;  /* buffer we're assigning to callback */
2744         kmutex_t        *hash_lock;
2745         arc_callback_t  *callback_list, *acb;
2746         int             freeable = FALSE;
2747 
2748         buf = zio->io_private;
2749         hdr = buf->b_hdr;
2750 
2751         /*
2752          * The hdr was inserted into hash-table and removed from lists
2753          * prior to starting I/O.  We should find this header, since
2754          * it's in the hash table, and it should be legit since it's
2755          * not possible to evict it during the I/O.  The only possible
2756          * reason for it not to be found is if we were freed during the
2757          * read.
2758          */
2759         found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2760             &hash_lock);
2761 
2762         ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2763             (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2764             (found == hdr && HDR_L2_READING(hdr)));
2765 
2766         hdr->b_flags &= ~ARC_L2_EVICTED;
2767         if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2768                 hdr->b_flags &= ~ARC_L2CACHE;
2769 
2770         /* byteswap if necessary */
2771         callback_list = hdr->b_acb;
2772         ASSERT(callback_list != NULL);
2773         if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2774                 dmu_object_byteswap_t bswap =
2775                     DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2776                 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2777                     byteswap_uint64_array :
2778                     dmu_ot_byteswap[bswap].ob_func;
2779                 func(buf->b_data, hdr->b_size);
2780         }
2781 
2782         arc_cksum_compute(buf, B_FALSE);
2783         arc_buf_watch(buf);
2784 
2785         if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2786                 /*
2787                  * Only call arc_access on anonymous buffers.  This is because
2788                  * if we've issued an I/O for an evicted buffer, we've already
2789                  * called arc_access (to prevent any simultaneous readers from
2790                  * getting confused).
2791                  */
2792                 arc_access(hdr, hash_lock);
2793         }
2794 
2795         /* create copies of the data buffer for the callers */
2796         abuf = buf;
2797         for (acb = callback_list; acb; acb = acb->acb_next) {
2798                 if (acb->acb_done) {
2799                         if (abuf == NULL) {
2800                                 ARCSTAT_BUMP(arcstat_duplicate_reads);
2801                                 abuf = arc_buf_clone(buf);
2802                         }
2803                         acb->acb_buf = abuf;
2804                         abuf = NULL;
2805                 }
2806         }
2807         hdr->b_acb = NULL;
2808         hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2809         ASSERT(!HDR_BUF_AVAILABLE(hdr));
2810         if (abuf == buf) {
2811                 ASSERT(buf->b_efunc == NULL);
2812                 ASSERT(hdr->b_datacnt == 1);
2813                 hdr->b_flags |= ARC_BUF_AVAILABLE;
2814         }
2815 
2816         ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2817 
2818         if (zio->io_error != 0) {
2819                 hdr->b_flags |= ARC_IO_ERROR;
2820                 if (hdr->b_state != arc_anon)
2821                         arc_change_state(arc_anon, hdr, hash_lock);
2822                 if (HDR_IN_HASH_TABLE(hdr))
2823                         buf_hash_remove(hdr);
2824                 freeable = refcount_is_zero(&hdr->b_refcnt);
2825         }
2826 
2827         /*
2828          * Broadcast before we drop the hash_lock to avoid the possibility
2829          * that the hdr (and hence the cv) might be freed before we get to
2830          * the cv_broadcast().
2831          */
2832         cv_broadcast(&hdr->b_cv);
2833 
2834         if (hash_lock) {
2835                 mutex_exit(hash_lock);
2836         } else {
2837                 /*
2838                  * This block was freed while we waited for the read to
2839                  * complete.  It has been removed from the hash table and
2840                  * moved to the anonymous state (so that it won't show up
2841                  * in the cache).
2842                  */
2843                 ASSERT3P(hdr->b_state, ==, arc_anon);
2844                 freeable = refcount_is_zero(&hdr->b_refcnt);
2845         }
2846 
2847         /* execute each callback and free its structure */
2848         while ((acb = callback_list) != NULL) {
2849                 if (acb->acb_done)
2850                         acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2851 
2852                 if (acb->acb_zio_dummy != NULL) {
2853                         acb->acb_zio_dummy->io_error = zio->io_error;
2854                         zio_nowait(acb->acb_zio_dummy);
2855                 }
2856 
2857                 callback_list = acb->acb_next;
2858                 kmem_free(acb, sizeof (arc_callback_t));
2859         }
2860 
2861         if (freeable)
2862                 arc_hdr_destroy(hdr);
2863 }
2864 
2865 /*
2866  * "Read" the block at the specified DVA (in bp) via the
2867  * cache.  If the block is found in the cache, invoke the provided
2868  * callback immediately and return.  Note that the `zio' parameter
2869  * in the callback will be NULL in this case, since no IO was
2870  * required.  If the block is not in the cache pass the read request
2871  * on to the spa with a substitute callback function, so that the
2872  * requested block will be added to the cache.
2873  *
2874  * If a read request arrives for a block that has a read in-progress,
2875  * either wait for the in-progress read to complete (and return the
2876  * results); or, if this is a read with a "done" func, add a record
2877  * to the read to invoke the "done" func when the read completes,
2878  * and return; or just return.
2879  *
2880  * arc_read_done() will invoke all the requested "done" functions
2881  * for readers of this block.
2882  */
2883 int
2884 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
2885     void *private, zio_priority_t priority, int zio_flags, uint32_t *arc_flags,
2886     const zbookmark_t *zb)
2887 {
2888         arc_buf_hdr_t *hdr;
2889         arc_buf_t *buf = NULL;
2890         kmutex_t *hash_lock;
2891         zio_t *rzio;
2892         uint64_t guid = spa_load_guid(spa);
2893 
2894 top:
2895         hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2896             &hash_lock);
2897         if (hdr && hdr->b_datacnt > 0) {
2898 
2899                 *arc_flags |= ARC_CACHED;
2900 
2901                 if (HDR_IO_IN_PROGRESS(hdr)) {
2902 
2903                         if (*arc_flags & ARC_WAIT) {
2904                                 cv_wait(&hdr->b_cv, hash_lock);
2905                                 mutex_exit(hash_lock);
2906                                 goto top;
2907                         }
2908                         ASSERT(*arc_flags & ARC_NOWAIT);
2909 
2910                         if (done) {
2911                                 arc_callback_t  *acb = NULL;
2912 
2913                                 acb = kmem_zalloc(sizeof (arc_callback_t),
2914                                     KM_SLEEP);
2915                                 acb->acb_done = done;
2916                                 acb->acb_private = private;
2917                                 if (pio != NULL)
2918                                         acb->acb_zio_dummy = zio_null(pio,
2919                                             spa, NULL, NULL, NULL, zio_flags);
2920 
2921                                 ASSERT(acb->acb_done != NULL);
2922                                 acb->acb_next = hdr->b_acb;
2923                                 hdr->b_acb = acb;
2924                                 add_reference(hdr, hash_lock, private);
2925                                 mutex_exit(hash_lock);
2926                                 return (0);
2927                         }
2928                         mutex_exit(hash_lock);
2929                         return (0);
2930                 }
2931 
2932                 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2933 
2934                 if (done) {
2935                         add_reference(hdr, hash_lock, private);
2936                         /*
2937                          * If this block is already in use, create a new
2938                          * copy of the data so that we will be guaranteed
2939                          * that arc_release() will always succeed.
2940                          */
2941                         buf = hdr->b_buf;
2942                         ASSERT(buf);
2943                         ASSERT(buf->b_data);
2944                         if (HDR_BUF_AVAILABLE(hdr)) {
2945                                 ASSERT(buf->b_efunc == NULL);
2946                                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2947                         } else {
2948                                 buf = arc_buf_clone(buf);
2949                         }
2950 
2951                 } else if (*arc_flags & ARC_PREFETCH &&
2952                     refcount_count(&hdr->b_refcnt) == 0) {
2953                         hdr->b_flags |= ARC_PREFETCH;
2954                 }
2955                 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
2956                 arc_access(hdr, hash_lock);
2957                 if (*arc_flags & ARC_L2CACHE)
2958                         hdr->b_flags |= ARC_L2CACHE;
2959                 if (*arc_flags & ARC_L2COMPRESS)
2960                         hdr->b_flags |= ARC_L2COMPRESS;
2961                 mutex_exit(hash_lock);
2962                 ARCSTAT_BUMP(arcstat_hits);
2963                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2964                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2965                     data, metadata, hits);
2966 
2967                 if (done)
2968                         done(NULL, buf, private);
2969         } else {
2970                 uint64_t size = BP_GET_LSIZE(bp);
2971                 arc_callback_t  *acb;
2972                 vdev_t *vd = NULL;
2973                 uint64_t addr = 0;
2974                 boolean_t devw = B_FALSE;
2975 
2976                 if (hdr == NULL) {
2977                         /* this block is not in the cache */
2978                         arc_buf_hdr_t   *exists;
2979                         arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2980                         buf = arc_buf_alloc(spa, size, private, type);
2981                         hdr = buf->b_hdr;
2982                         hdr->b_dva = *BP_IDENTITY(bp);
2983                         hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2984                         hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2985                         exists = buf_hash_insert(hdr, &hash_lock);
2986                         if (exists) {
2987                                 /* somebody beat us to the hash insert */
2988                                 mutex_exit(hash_lock);
2989                                 buf_discard_identity(hdr);
2990                                 (void) arc_buf_remove_ref(buf, private);
2991                                 goto top; /* restart the IO request */
2992                         }
2993                         /* if this is a prefetch, we don't have a reference */
2994                         if (*arc_flags & ARC_PREFETCH) {
2995                                 (void) remove_reference(hdr, hash_lock,
2996                                     private);
2997                                 hdr->b_flags |= ARC_PREFETCH;
2998                         }
2999                         if (*arc_flags & ARC_L2CACHE)
3000                                 hdr->b_flags |= ARC_L2CACHE;
3001                         if (*arc_flags & ARC_L2COMPRESS)
3002                                 hdr->b_flags |= ARC_L2COMPRESS;
3003                         if (BP_GET_LEVEL(bp) > 0)
3004                                 hdr->b_flags |= ARC_INDIRECT;
3005                 } else {
3006                         /* this block is in the ghost cache */
3007                         ASSERT(GHOST_STATE(hdr->b_state));
3008                         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3009                         ASSERT0(refcount_count(&hdr->b_refcnt));
3010                         ASSERT(hdr->b_buf == NULL);
3011 
3012                         /* if this is a prefetch, we don't have a reference */
3013                         if (*arc_flags & ARC_PREFETCH)
3014                                 hdr->b_flags |= ARC_PREFETCH;
3015                         else
3016                                 add_reference(hdr, hash_lock, private);
3017                         if (*arc_flags & ARC_L2CACHE)
3018                                 hdr->b_flags |= ARC_L2CACHE;
3019                         if (*arc_flags & ARC_L2COMPRESS)
3020                                 hdr->b_flags |= ARC_L2COMPRESS;
3021                         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
3022                         buf->b_hdr = hdr;
3023                         buf->b_data = NULL;
3024                         buf->b_efunc = NULL;
3025                         buf->b_private = NULL;
3026                         buf->b_next = NULL;
3027                         hdr->b_buf = buf;
3028                         ASSERT(hdr->b_datacnt == 0);
3029                         hdr->b_datacnt = 1;
3030                         arc_get_data_buf(buf);
3031                         arc_access(hdr, hash_lock);
3032                 }
3033 
3034                 ASSERT(!GHOST_STATE(hdr->b_state));
3035 
3036                 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
3037                 acb->acb_done = done;
3038                 acb->acb_private = private;
3039 
3040                 ASSERT(hdr->b_acb == NULL);
3041                 hdr->b_acb = acb;
3042                 hdr->b_flags |= ARC_IO_IN_PROGRESS;
3043 
3044                 if (HDR_L2CACHE(hdr) && hdr->b_l2hdr != NULL &&
3045                     (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
3046                         devw = hdr->b_l2hdr->b_dev->l2ad_writing;
3047                         addr = hdr->b_l2hdr->b_daddr;
3048                         /*
3049                          * Lock out device removal.
3050                          */
3051                         if (vdev_is_dead(vd) ||
3052                             !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
3053                                 vd = NULL;
3054                 }
3055 
3056                 mutex_exit(hash_lock);
3057 
3058                 /*
3059                  * At this point, we have a level 1 cache miss.  Try again in
3060                  * L2ARC if possible.
3061                  */
3062                 ASSERT3U(hdr->b_size, ==, size);
3063                 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
3064                     uint64_t, size, zbookmark_t *, zb);
3065                 ARCSTAT_BUMP(arcstat_misses);
3066                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
3067                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
3068                     data, metadata, misses);
3069 
3070                 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
3071                         /*
3072                          * Read from the L2ARC if the following are true:
3073                          * 1. The L2ARC vdev was previously cached.
3074                          * 2. This buffer still has L2ARC metadata.
3075                          * 3. This buffer isn't currently writing to the L2ARC.
3076                          * 4. The L2ARC entry wasn't evicted, which may
3077                          *    also have invalidated the vdev.
3078                          * 5. This isn't prefetch and l2arc_noprefetch is set.
3079                          */
3080                         if (hdr->b_l2hdr != NULL &&
3081                             !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
3082                             !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
3083                                 l2arc_read_callback_t *cb;
3084 
3085                                 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
3086                                 ARCSTAT_BUMP(arcstat_l2_hits);
3087 
3088                                 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
3089                                     KM_SLEEP);
3090                                 cb->l2rcb_buf = buf;
3091                                 cb->l2rcb_spa = spa;
3092                                 cb->l2rcb_bp = *bp;
3093                                 cb->l2rcb_zb = *zb;
3094                                 cb->l2rcb_flags = zio_flags;
3095                                 cb->l2rcb_compress = hdr->b_l2hdr->b_compress;
3096 
3097                                 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
3098                                     addr + size < vd->vdev_psize -
3099                                     VDEV_LABEL_END_SIZE);
3100 
3101                                 /*
3102                                  * l2arc read.  The SCL_L2ARC lock will be
3103                                  * released by l2arc_read_done().
3104                                  * Issue a null zio if the underlying buffer
3105                                  * was squashed to zero size by compression.
3106                                  */
3107                                 if (hdr->b_l2hdr->b_compress ==
3108                                     ZIO_COMPRESS_EMPTY) {
3109                                         rzio = zio_null(pio, spa, vd,
3110                                             l2arc_read_done, cb,
3111                                             zio_flags | ZIO_FLAG_DONT_CACHE |
3112                                             ZIO_FLAG_CANFAIL |
3113                                             ZIO_FLAG_DONT_PROPAGATE |
3114                                             ZIO_FLAG_DONT_RETRY);
3115                                 } else {
3116                                         rzio = zio_read_phys(pio, vd, addr,
3117                                             hdr->b_l2hdr->b_asize,
3118                                             buf->b_data, ZIO_CHECKSUM_OFF,
3119                                             l2arc_read_done, cb, priority,
3120                                             zio_flags | ZIO_FLAG_DONT_CACHE |
3121                                             ZIO_FLAG_CANFAIL |
3122                                             ZIO_FLAG_DONT_PROPAGATE |
3123                                             ZIO_FLAG_DONT_RETRY, B_FALSE);
3124                                 }
3125                                 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
3126                                     zio_t *, rzio);
3127                                 ARCSTAT_INCR(arcstat_l2_read_bytes,
3128                                     hdr->b_l2hdr->b_asize);
3129 
3130                                 if (*arc_flags & ARC_NOWAIT) {
3131                                         zio_nowait(rzio);
3132                                         return (0);
3133                                 }
3134 
3135                                 ASSERT(*arc_flags & ARC_WAIT);
3136                                 if (zio_wait(rzio) == 0)
3137                                         return (0);
3138 
3139                                 /* l2arc read error; goto zio_read() */
3140                         } else {
3141                                 DTRACE_PROBE1(l2arc__miss,
3142                                     arc_buf_hdr_t *, hdr);
3143                                 ARCSTAT_BUMP(arcstat_l2_misses);
3144                                 if (HDR_L2_WRITING(hdr))
3145                                         ARCSTAT_BUMP(arcstat_l2_rw_clash);
3146                                 spa_config_exit(spa, SCL_L2ARC, vd);
3147                         }
3148                 } else {
3149                         if (vd != NULL)
3150                                 spa_config_exit(spa, SCL_L2ARC, vd);
3151                         if (l2arc_ndev != 0) {
3152                                 DTRACE_PROBE1(l2arc__miss,
3153                                     arc_buf_hdr_t *, hdr);
3154                                 ARCSTAT_BUMP(arcstat_l2_misses);
3155                         }
3156                 }
3157 
3158                 rzio = zio_read(pio, spa, bp, buf->b_data, size,
3159                     arc_read_done, buf, priority, zio_flags, zb);
3160 
3161                 if (*arc_flags & ARC_WAIT)
3162                         return (zio_wait(rzio));
3163 
3164                 ASSERT(*arc_flags & ARC_NOWAIT);
3165                 zio_nowait(rzio);
3166         }
3167         return (0);
3168 }
3169 
3170 void
3171 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
3172 {
3173         ASSERT(buf->b_hdr != NULL);
3174         ASSERT(buf->b_hdr->b_state != arc_anon);
3175         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
3176         ASSERT(buf->b_efunc == NULL);
3177         ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
3178 
3179         buf->b_efunc = func;
3180         buf->b_private = private;
3181 }
3182 
3183 /*
3184  * Notify the arc that a block was freed, and thus will never be used again.
3185  */
3186 void
3187 arc_freed(spa_t *spa, const blkptr_t *bp)
3188 {
3189         arc_buf_hdr_t *hdr;
3190         kmutex_t *hash_lock;
3191         uint64_t guid = spa_load_guid(spa);
3192 
3193         hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
3194             &hash_lock);
3195         if (hdr == NULL)
3196                 return;
3197         if (HDR_BUF_AVAILABLE(hdr)) {
3198                 arc_buf_t *buf = hdr->b_buf;
3199                 add_reference(hdr, hash_lock, FTAG);
3200                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3201                 mutex_exit(hash_lock);
3202 
3203                 arc_release(buf, FTAG);
3204                 (void) arc_buf_remove_ref(buf, FTAG);
3205         } else {
3206                 mutex_exit(hash_lock);
3207         }
3208 
3209 }
3210 
3211 /*
3212  * This is used by the DMU to let the ARC know that a buffer is
3213  * being evicted, so the ARC should clean up.  If this arc buf
3214  * is not yet in the evicted state, it will be put there.
3215  */
3216 int
3217 arc_buf_evict(arc_buf_t *buf)
3218 {
3219         arc_buf_hdr_t *hdr;
3220         kmutex_t *hash_lock;
3221         arc_buf_t **bufp;
3222 
3223         mutex_enter(&buf->b_evict_lock);
3224         hdr = buf->b_hdr;
3225         if (hdr == NULL) {
3226                 /*
3227                  * We are in arc_do_user_evicts().
3228                  */
3229                 ASSERT(buf->b_data == NULL);
3230                 mutex_exit(&buf->b_evict_lock);
3231                 return (0);
3232         } else if (buf->b_data == NULL) {
3233                 arc_buf_t copy = *buf; /* structure assignment */
3234                 /*
3235                  * We are on the eviction list; process this buffer now
3236                  * but let arc_do_user_evicts() do the reaping.
3237                  */
3238                 buf->b_efunc = NULL;
3239                 mutex_exit(&buf->b_evict_lock);
3240                 VERIFY(copy.b_efunc(&copy) == 0);
3241                 return (1);
3242         }
3243         hash_lock = HDR_LOCK(hdr);
3244         mutex_enter(hash_lock);
3245         hdr = buf->b_hdr;
3246         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3247 
3248         ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
3249         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3250 
3251         /*
3252          * Pull this buffer off of the hdr
3253          */
3254         bufp = &hdr->b_buf;
3255         while (*bufp != buf)
3256                 bufp = &(*bufp)->b_next;
3257         *bufp = buf->b_next;
3258 
3259         ASSERT(buf->b_data != NULL);
3260         arc_buf_destroy(buf, FALSE, FALSE);
3261 
3262         if (hdr->b_datacnt == 0) {
3263                 arc_state_t *old_state = hdr->b_state;
3264                 arc_state_t *evicted_state;
3265 
3266                 ASSERT(hdr->b_buf == NULL);
3267                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
3268 
3269                 evicted_state =
3270                     (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3271 
3272                 mutex_enter(&old_state->arcs_mtx);
3273                 mutex_enter(&evicted_state->arcs_mtx);
3274 
3275                 arc_change_state(evicted_state, hdr, hash_lock);
3276                 ASSERT(HDR_IN_HASH_TABLE(hdr));
3277                 hdr->b_flags |= ARC_IN_HASH_TABLE;
3278                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3279 
3280                 mutex_exit(&evicted_state->arcs_mtx);
3281                 mutex_exit(&old_state->arcs_mtx);
3282         }
3283         mutex_exit(hash_lock);
3284         mutex_exit(&buf->b_evict_lock);
3285 
3286         VERIFY(buf->b_efunc(buf) == 0);
3287         buf->b_efunc = NULL;
3288         buf->b_private = NULL;
3289         buf->b_hdr = NULL;
3290         buf->b_next = NULL;
3291         kmem_cache_free(buf_cache, buf);
3292         return (1);
3293 }
3294 
3295 /*
3296  * Release this buffer from the cache, making it an anonymous buffer.  This
3297  * must be done after a read and prior to modifying the buffer contents.
3298  * If the buffer has more than one reference, we must make
3299  * a new hdr for the buffer.
3300  */
3301 void
3302 arc_release(arc_buf_t *buf, void *tag)
3303 {
3304         arc_buf_hdr_t *hdr;
3305         kmutex_t *hash_lock = NULL;
3306         l2arc_buf_hdr_t *l2hdr;
3307         uint64_t buf_size;
3308 
3309         /*
3310          * It would be nice to assert that if it's DMU metadata (level >
3311          * 0 || it's the dnode file), then it must be syncing context.
3312          * But we don't know that information at this level.
3313          */
3314 
3315         mutex_enter(&buf->b_evict_lock);
3316         hdr = buf->b_hdr;
3317 
3318         /* this buffer is not on any list */
3319         ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3320 
3321         if (hdr->b_state == arc_anon) {
3322                 /* this buffer is already released */
3323                 ASSERT(buf->b_efunc == NULL);
3324         } else {
3325                 hash_lock = HDR_LOCK(hdr);
3326                 mutex_enter(hash_lock);
3327                 hdr = buf->b_hdr;
3328                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3329         }
3330 
3331         l2hdr = hdr->b_l2hdr;
3332         if (l2hdr) {
3333                 mutex_enter(&l2arc_buflist_mtx);
3334                 hdr->b_l2hdr = NULL;
3335         }
3336         buf_size = hdr->b_size;
3337 
3338         /*
3339          * Do we have more than one buf?
3340          */
3341         if (hdr->b_datacnt > 1) {
3342                 arc_buf_hdr_t *nhdr;
3343                 arc_buf_t **bufp;
3344                 uint64_t blksz = hdr->b_size;
3345                 uint64_t spa = hdr->b_spa;
3346                 arc_buf_contents_t type = hdr->b_type;
3347                 uint32_t flags = hdr->b_flags;
3348 
3349                 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3350                 /*
3351                  * Pull the data off of this hdr and attach it to
3352                  * a new anonymous hdr.
3353                  */
3354                 (void) remove_reference(hdr, hash_lock, tag);
3355                 bufp = &hdr->b_buf;
3356                 while (*bufp != buf)
3357                         bufp = &(*bufp)->b_next;
3358                 *bufp = buf->b_next;
3359                 buf->b_next = NULL;
3360 
3361                 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3362                 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3363                 if (refcount_is_zero(&hdr->b_refcnt)) {
3364                         uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3365                         ASSERT3U(*size, >=, hdr->b_size);
3366                         atomic_add_64(size, -hdr->b_size);
3367                 }
3368 
3369                 /*
3370                  * We're releasing a duplicate user data buffer, update
3371                  * our statistics accordingly.
3372                  */
3373                 if (hdr->b_type == ARC_BUFC_DATA) {
3374                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
3375                         ARCSTAT_INCR(arcstat_duplicate_buffers_size,
3376                             -hdr->b_size);
3377                 }
3378                 hdr->b_datacnt -= 1;
3379                 arc_cksum_verify(buf);
3380                 arc_buf_unwatch(buf);
3381 
3382                 mutex_exit(hash_lock);
3383 
3384                 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3385                 nhdr->b_size = blksz;
3386                 nhdr->b_spa = spa;
3387                 nhdr->b_type = type;
3388                 nhdr->b_buf = buf;
3389                 nhdr->b_state = arc_anon;
3390                 nhdr->b_arc_access = 0;
3391                 nhdr->b_flags = flags & ARC_L2_WRITING;
3392                 nhdr->b_l2hdr = NULL;
3393                 nhdr->b_datacnt = 1;
3394                 nhdr->b_freeze_cksum = NULL;
3395                 (void) refcount_add(&nhdr->b_refcnt, tag);
3396                 buf->b_hdr = nhdr;
3397                 mutex_exit(&buf->b_evict_lock);
3398                 atomic_add_64(&arc_anon->arcs_size, blksz);
3399         } else {
3400                 mutex_exit(&buf->b_evict_lock);
3401                 ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3402                 ASSERT(!list_link_active(&hdr->b_arc_node));
3403                 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3404                 if (hdr->b_state != arc_anon)
3405                         arc_change_state(arc_anon, hdr, hash_lock);
3406                 hdr->b_arc_access = 0;
3407                 if (hash_lock)
3408                         mutex_exit(hash_lock);
3409 
3410                 buf_discard_identity(hdr);
3411                 arc_buf_thaw(buf);
3412         }
3413         buf->b_efunc = NULL;
3414         buf->b_private = NULL;
3415 
3416         if (l2hdr) {
3417                 ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
3418                 list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3419                 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3420                 ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3421                 mutex_exit(&l2arc_buflist_mtx);
3422         }
3423 }
3424 
3425 int
3426 arc_released(arc_buf_t *buf)
3427 {
3428         int released;
3429 
3430         mutex_enter(&buf->b_evict_lock);
3431         released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
3432         mutex_exit(&buf->b_evict_lock);
3433         return (released);
3434 }
3435 
3436 int
3437 arc_has_callback(arc_buf_t *buf)
3438 {
3439         int callback;
3440 
3441         mutex_enter(&buf->b_evict_lock);
3442         callback = (buf->b_efunc != NULL);
3443         mutex_exit(&buf->b_evict_lock);
3444         return (callback);
3445 }
3446 
3447 #ifdef ZFS_DEBUG
3448 int
3449 arc_referenced(arc_buf_t *buf)
3450 {
3451         int referenced;
3452 
3453         mutex_enter(&buf->b_evict_lock);
3454         referenced = (refcount_count(&buf->b_hdr->b_refcnt));
3455         mutex_exit(&buf->b_evict_lock);
3456         return (referenced);
3457 }
3458 #endif
3459 
3460 static void
3461 arc_write_ready(zio_t *zio)
3462 {
3463         arc_write_callback_t *callback = zio->io_private;
3464         arc_buf_t *buf = callback->awcb_buf;
3465         arc_buf_hdr_t *hdr = buf->b_hdr;
3466 
3467         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3468         callback->awcb_ready(zio, buf, callback->awcb_private);
3469 
3470         /*
3471          * If the IO is already in progress, then this is a re-write
3472          * attempt, so we need to thaw and re-compute the cksum.
3473          * It is the responsibility of the callback to handle the
3474          * accounting for any re-write attempt.
3475          */
3476         if (HDR_IO_IN_PROGRESS(hdr)) {
3477                 mutex_enter(&hdr->b_freeze_lock);
3478                 if (hdr->b_freeze_cksum != NULL) {
3479                         kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3480                         hdr->b_freeze_cksum = NULL;
3481                 }
3482                 mutex_exit(&hdr->b_freeze_lock);
3483         }
3484         arc_cksum_compute(buf, B_FALSE);
3485         hdr->b_flags |= ARC_IO_IN_PROGRESS;
3486 }
3487 
3488 /*
3489  * The SPA calls this callback for each physical write that happens on behalf
3490  * of a logical write.  See the comment in dbuf_write_physdone() for details.
3491  */
3492 static void
3493 arc_write_physdone(zio_t *zio)
3494 {
3495         arc_write_callback_t *cb = zio->io_private;
3496         if (cb->awcb_physdone != NULL)
3497                 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
3498 }
3499 
3500 static void
3501 arc_write_done(zio_t *zio)
3502 {
3503         arc_write_callback_t *callback = zio->io_private;
3504         arc_buf_t *buf = callback->awcb_buf;
3505         arc_buf_hdr_t *hdr = buf->b_hdr;
3506 
3507         ASSERT(hdr->b_acb == NULL);
3508 
3509         if (zio->io_error == 0) {
3510                 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3511                 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3512                 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3513         } else {
3514                 ASSERT(BUF_EMPTY(hdr));
3515         }
3516 
3517         /*
3518          * If the block to be written was all-zero, we may have
3519          * compressed it away.  In this case no write was performed
3520          * so there will be no dva/birth/checksum.  The buffer must
3521          * therefore remain anonymous (and uncached).
3522          */
3523         if (!BUF_EMPTY(hdr)) {
3524                 arc_buf_hdr_t *exists;
3525                 kmutex_t *hash_lock;
3526 
3527                 ASSERT(zio->io_error == 0);
3528 
3529                 arc_cksum_verify(buf);
3530 
3531                 exists = buf_hash_insert(hdr, &hash_lock);
3532                 if (exists) {
3533                         /*
3534                          * This can only happen if we overwrite for
3535                          * sync-to-convergence, because we remove
3536                          * buffers from the hash table when we arc_free().
3537                          */
3538                         if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3539                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3540                                         panic("bad overwrite, hdr=%p exists=%p",
3541                                             (void *)hdr, (void *)exists);
3542                                 ASSERT(refcount_is_zero(&exists->b_refcnt));
3543                                 arc_change_state(arc_anon, exists, hash_lock);
3544                                 mutex_exit(hash_lock);
3545                                 arc_hdr_destroy(exists);
3546                                 exists = buf_hash_insert(hdr, &hash_lock);
3547                                 ASSERT3P(exists, ==, NULL);
3548                         } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
3549                                 /* nopwrite */
3550                                 ASSERT(zio->io_prop.zp_nopwrite);
3551                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3552                                         panic("bad nopwrite, hdr=%p exists=%p",
3553                                             (void *)hdr, (void *)exists);
3554                         } else {
3555                                 /* Dedup */
3556                                 ASSERT(hdr->b_datacnt == 1);
3557                                 ASSERT(hdr->b_state == arc_anon);
3558                                 ASSERT(BP_GET_DEDUP(zio->io_bp));
3559                                 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3560                         }
3561                 }
3562                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3563                 /* if it's not anon, we are doing a scrub */
3564                 if (!exists && hdr->b_state == arc_anon)
3565                         arc_access(hdr, hash_lock);
3566                 mutex_exit(hash_lock);
3567         } else {
3568                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3569         }
3570 
3571         ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3572         callback->awcb_done(zio, buf, callback->awcb_private);
3573 
3574         kmem_free(callback, sizeof (arc_write_callback_t));
3575 }
3576 
3577 zio_t *
3578 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3579     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
3580     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
3581     arc_done_func_t *done, void *private, zio_priority_t priority,
3582     int zio_flags, const zbookmark_t *zb)
3583 {
3584         arc_buf_hdr_t *hdr = buf->b_hdr;
3585         arc_write_callback_t *callback;
3586         zio_t *zio;
3587 
3588         ASSERT(ready != NULL);
3589         ASSERT(done != NULL);
3590         ASSERT(!HDR_IO_ERROR(hdr));
3591         ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3592         ASSERT(hdr->b_acb == NULL);
3593         if (l2arc)
3594                 hdr->b_flags |= ARC_L2CACHE;
3595         if (l2arc_compress)
3596                 hdr->b_flags |= ARC_L2COMPRESS;
3597         callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3598         callback->awcb_ready = ready;
3599         callback->awcb_physdone = physdone;
3600         callback->awcb_done = done;
3601         callback->awcb_private = private;
3602         callback->awcb_buf = buf;
3603 
3604         zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3605             arc_write_ready, arc_write_physdone, arc_write_done, callback,
3606             priority, zio_flags, zb);
3607 
3608         return (zio);
3609 }
3610 
3611 static int
3612 arc_memory_throttle(uint64_t reserve, uint64_t txg)
3613 {
3614 #ifdef _KERNEL
3615         uint64_t available_memory = ptob(freemem);
3616         static uint64_t page_load = 0;
3617         static uint64_t last_txg = 0;
3618 
3619 #if defined(__i386)
3620         available_memory =
3621             MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3622 #endif
3623 
3624         if (freemem > physmem * arc_lotsfree_percent / 100)
3625                 return (0);
3626 
3627         if (txg > last_txg) {
3628                 last_txg = txg;
3629                 page_load = 0;
3630         }
3631         /*
3632          * If we are in pageout, we know that memory is already tight,
3633          * the arc is already going to be evicting, so we just want to
3634          * continue to let page writes occur as quickly as possible.
3635          */
3636         if (curproc == proc_pageout) {
3637                 if (page_load > MAX(ptob(minfree), available_memory) / 4)
3638                         return (SET_ERROR(ERESTART));
3639                 /* Note: reserve is inflated, so we deflate */
3640                 page_load += reserve / 8;
3641                 return (0);
3642         } else if (page_load > 0 && arc_reclaim_needed()) {
3643                 /* memory is low, delay before restarting */
3644                 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3645                 return (SET_ERROR(EAGAIN));
3646         }
3647         page_load = 0;
3648 #endif
3649         return (0);
3650 }
3651 
3652 void
3653 arc_tempreserve_clear(uint64_t reserve)
3654 {
3655         atomic_add_64(&arc_tempreserve, -reserve);
3656         ASSERT((int64_t)arc_tempreserve >= 0);
3657 }
3658 
3659 int
3660 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3661 {
3662         int error;
3663         uint64_t anon_size;
3664 
3665         if (reserve > arc_c/4 && !arc_no_grow)
3666                 arc_c = MIN(arc_c_max, reserve * 4);
3667         if (reserve > arc_c)
3668                 return (SET_ERROR(ENOMEM));
3669 
3670         /*
3671          * Don't count loaned bufs as in flight dirty data to prevent long
3672          * network delays from blocking transactions that are ready to be
3673          * assigned to a txg.
3674          */
3675         anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3676 
3677         /*
3678          * Writes will, almost always, require additional memory allocations
3679          * in order to compress/encrypt/etc the data.  We therefore need to
3680          * make sure that there is sufficient available memory for this.
3681          */
3682         error = arc_memory_throttle(reserve, txg);
3683         if (error != 0)
3684                 return (error);
3685 
3686         /*
3687          * Throttle writes when the amount of dirty data in the cache
3688          * gets too large.  We try to keep the cache less than half full
3689          * of dirty blocks so that our sync times don't grow too large.
3690          * Note: if two requests come in concurrently, we might let them
3691          * both succeed, when one of them should fail.  Not a huge deal.
3692          */
3693 
3694         if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3695             anon_size > arc_c / 4) {
3696                 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3697                     "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3698                     arc_tempreserve>>10,
3699                     arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3700                     arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3701                     reserve>>10, arc_c>>10);
3702                 return (SET_ERROR(ERESTART));
3703         }
3704         atomic_add_64(&arc_tempreserve, reserve);
3705         return (0);
3706 }
3707 
3708 void
3709 arc_init(void)
3710 {
3711         mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3712         cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3713 
3714         /* Convert seconds to clock ticks */
3715         arc_min_prefetch_lifespan = 1 * hz;
3716 
3717         /* Start out with 1/8 of all memory */
3718         arc_c = physmem * PAGESIZE / 8;
3719 
3720 #ifdef _KERNEL
3721         /*
3722          * On architectures where the physical memory can be larger
3723          * than the addressable space (intel in 32-bit mode), we may
3724          * need to limit the cache to 1/8 of VM size.
3725          */
3726         arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3727 #endif
3728 
3729         /* set min cache to 1/32 of all memory, or 64MB, whichever is more */
3730         arc_c_min = MAX(arc_c / 4, 64<<20);
3731         /* set max to 3/4 of all memory, or all but 1GB, whichever is more */
3732         if (arc_c * 8 >= 1<<30)
3733                 arc_c_max = (arc_c * 8) - (1<<30);
3734         else
3735                 arc_c_max = arc_c_min;
3736         arc_c_max = MAX(arc_c * 6, arc_c_max);
3737 
3738         /*
3739          * Allow the tunables to override our calculations if they are
3740          * reasonable (ie. over 64MB)
3741          */
3742         if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
3743                 arc_c_max = zfs_arc_max;
3744         if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
3745                 arc_c_min = zfs_arc_min;
3746 
3747         arc_c = arc_c_max;
3748         arc_p = (arc_c >> 1);
3749 
3750         /* limit meta-data to 1/4 of the arc capacity */
3751         arc_meta_limit = arc_c_max / 4;
3752 
3753         /* Allow the tunable to override if it is reasonable */
3754         if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
3755                 arc_meta_limit = zfs_arc_meta_limit;
3756 
3757         if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
3758                 arc_c_min = arc_meta_limit / 2;
3759 
3760         if (zfs_arc_grow_retry > 0)
3761                 arc_grow_retry = zfs_arc_grow_retry;
3762 
3763         if (zfs_arc_shrink_shift > 0)
3764                 arc_shrink_shift = zfs_arc_shrink_shift;
3765 
3766         if (zfs_arc_p_min_shift > 0)
3767                 arc_p_min_shift = zfs_arc_p_min_shift;
3768 
3769         /* if kmem_flags are set, lets try to use less memory */
3770         if (kmem_debugging())
3771                 arc_c = arc_c / 2;
3772         if (arc_c < arc_c_min)
3773                 arc_c = arc_c_min;
3774 
3775         arc_anon = &ARC_anon;
3776         arc_mru = &ARC_mru;
3777         arc_mru_ghost = &ARC_mru_ghost;
3778         arc_mfu = &ARC_mfu;
3779         arc_mfu_ghost = &ARC_mfu_ghost;
3780         arc_l2c_only = &ARC_l2c_only;
3781         arc_size = 0;
3782 
3783         mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3784         mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3785         mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3786         mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3787         mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3788         mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3789 
3790         list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
3791             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3792         list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
3793             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3794         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
3795             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3796         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
3797             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3798         list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
3799             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3800         list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
3801             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3802         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
3803             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3804         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
3805             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3806         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3807             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3808         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3809             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3810 
3811         buf_init();
3812 
3813         arc_thread_exit = 0;
3814         arc_eviction_list = NULL;
3815         mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
3816         bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3817 
3818         arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
3819             sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
3820 
3821         if (arc_ksp != NULL) {
3822                 arc_ksp->ks_data = &arc_stats;
3823                 kstat_install(arc_ksp);
3824         }
3825 
3826         (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3827             TS_RUN, minclsyspri);
3828 
3829         arc_dead = FALSE;
3830         arc_warm = B_FALSE;
3831 
3832         /*
3833          * Calculate maximum amount of dirty data per pool.
3834          *
3835          * If it has been set by /etc/system, take that.
3836          * Otherwise, use a percentage of physical memory defined by
3837          * zfs_dirty_data_max_percent (default 10%) with a cap at
3838          * zfs_dirty_data_max_max (default 4GB).
3839          */
3840         if (zfs_dirty_data_max == 0) {
3841                 zfs_dirty_data_max = physmem * PAGESIZE *
3842                     zfs_dirty_data_max_percent / 100;
3843                 zfs_dirty_data_max = MIN(zfs_dirty_data_max,
3844                     zfs_dirty_data_max_max);
3845         }
3846 }
3847 
3848 void
3849 arc_fini(void)
3850 {
3851         mutex_enter(&arc_reclaim_thr_lock);
3852         arc_thread_exit = 1;
3853         while (arc_thread_exit != 0)
3854                 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3855         mutex_exit(&arc_reclaim_thr_lock);
3856 
3857         arc_flush(NULL);
3858 
3859         arc_dead = TRUE;
3860 
3861         if (arc_ksp != NULL) {
3862                 kstat_delete(arc_ksp);
3863                 arc_ksp = NULL;
3864         }
3865 
3866         mutex_destroy(&arc_eviction_mtx);
3867         mutex_destroy(&arc_reclaim_thr_lock);
3868         cv_destroy(&arc_reclaim_thr_cv);
3869 
3870         list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
3871         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
3872         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
3873         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
3874         list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
3875         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
3876         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
3877         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3878 
3879         mutex_destroy(&arc_anon->arcs_mtx);
3880         mutex_destroy(&arc_mru->arcs_mtx);
3881         mutex_destroy(&arc_mru_ghost->arcs_mtx);
3882         mutex_destroy(&arc_mfu->arcs_mtx);
3883         mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3884         mutex_destroy(&arc_l2c_only->arcs_mtx);
3885 
3886         buf_fini();
3887 
3888         ASSERT(arc_loaned_bytes == 0);
3889 }
3890 
3891 /*
3892  * Level 2 ARC
3893  *
3894  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3895  * It uses dedicated storage devices to hold cached data, which are populated
3896  * using large infrequent writes.  The main role of this cache is to boost
3897  * the performance of random read workloads.  The intended L2ARC devices
3898  * include short-stroked disks, solid state disks, and other media with
3899  * substantially faster read latency than disk.
3900  *
3901  *                 +-----------------------+
3902  *                 |         ARC           |
3903  *                 +-----------------------+
3904  *                    |         ^     ^
3905  *                    |         |     |
3906  *      l2arc_feed_thread()    arc_read()
3907  *                    |         |     |
3908  *                    |  l2arc read   |
3909  *                    V         |     |
3910  *               +---------------+    |
3911  *               |     L2ARC     |    |
3912  *               +---------------+    |
3913  *                   |    ^           |
3914  *          l2arc_write() |           |
3915  *                   |    |           |
3916  *                   V    |           |
3917  *                 +-------+      +-------+
3918  *                 | vdev  |      | vdev  |
3919  *                 | cache |      | cache |
3920  *                 +-------+      +-------+
3921  *                 +=========+     .-----.
3922  *                 :  L2ARC  :    |-_____-|
3923  *                 : devices :    | Disks |
3924  *                 +=========+    `-_____-'
3925  *
3926  * Read requests are satisfied from the following sources, in order:
3927  *
3928  *      1) ARC
3929  *      2) vdev cache of L2ARC devices
3930  *      3) L2ARC devices
3931  *      4) vdev cache of disks
3932  *      5) disks
3933  *
3934  * Some L2ARC device types exhibit extremely slow write performance.
3935  * To accommodate for this there are some significant differences between
3936  * the L2ARC and traditional cache design:
3937  *
3938  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3939  * the ARC behave as usual, freeing buffers and placing headers on ghost
3940  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3941  * this would add inflated write latencies for all ARC memory pressure.
3942  *
3943  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3944  * It does this by periodically scanning buffers from the eviction-end of
3945  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3946  * not already there. It scans until a headroom of buffers is satisfied,
3947  * which itself is a buffer for ARC eviction. If a compressible buffer is
3948  * found during scanning and selected for writing to an L2ARC device, we
3949  * temporarily boost scanning headroom during the next scan cycle to make
3950  * sure we adapt to compression effects (which might significantly reduce
3951  * the data volume we write to L2ARC). The thread that does this is
3952  * l2arc_feed_thread(), illustrated below; example sizes are included to
3953  * provide a better sense of ratio than this diagram:
3954  *
3955  *             head -->                        tail
3956  *              +---------------------+----------+
3957  *      ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3958  *              +---------------------+----------+   |   o L2ARC eligible
3959  *      ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3960  *              +---------------------+----------+   |
3961  *                   15.9 Gbytes      ^ 32 Mbytes    |
3962  *                                 headroom          |
3963  *                                            l2arc_feed_thread()
3964  *                                                   |
3965  *                       l2arc write hand <--[oooo]--'
3966  *                               |           8 Mbyte
3967  *                               |          write max
3968  *                               V
3969  *                +==============================+
3970  *      L2ARC dev |####|#|###|###|    |####| ... |
3971  *                +==============================+
3972  *                           32 Gbytes
3973  *
3974  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3975  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3976  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3977  * safe to say that this is an uncommon case, since buffers at the end of
3978  * the ARC lists have moved there due to inactivity.
3979  *
3980  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3981  * then the L2ARC simply misses copying some buffers.  This serves as a
3982  * pressure valve to prevent heavy read workloads from both stalling the ARC
3983  * with waits and clogging the L2ARC with writes.  This also helps prevent
3984  * the potential for the L2ARC to churn if it attempts to cache content too
3985  * quickly, such as during backups of the entire pool.
3986  *
3987  * 5. After system boot and before the ARC has filled main memory, there are
3988  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
3989  * lists can remain mostly static.  Instead of searching from tail of these
3990  * lists as pictured, the l2arc_feed_thread() will search from the list heads
3991  * for eligible buffers, greatly increasing its chance of finding them.
3992  *
3993  * The L2ARC device write speed is also boosted during this time so that
3994  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
3995  * there are no L2ARC reads, and no fear of degrading read performance
3996  * through increased writes.
3997  *
3998  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
3999  * the vdev queue can aggregate them into larger and fewer writes.  Each
4000  * device is written to in a rotor fashion, sweeping writes through
4001  * available space then repeating.
4002  *
4003  * 7. The L2ARC does not store dirty content.  It never needs to flush
4004  * write buffers back to disk based storage.
4005  *
4006  * 8. If an ARC buffer is written (and dirtied) which also exists in the
4007  * L2ARC, the now stale L2ARC buffer is immediately dropped.
4008  *
4009  * The performance of the L2ARC can be tweaked by a number of tunables, which
4010  * may be necessary for different workloads:
4011  *
4012  *      l2arc_write_max         max write bytes per interval
4013  *      l2arc_write_boost       extra write bytes during device warmup
4014  *      l2arc_noprefetch        skip caching prefetched buffers
4015  *      l2arc_headroom          number of max device writes to precache
4016  *      l2arc_headroom_boost    when we find compressed buffers during ARC
4017  *                              scanning, we multiply headroom by this
4018  *                              percentage factor for the next scan cycle,
4019  *                              since more compressed buffers are likely to
4020  *                              be present
4021  *      l2arc_feed_secs         seconds between L2ARC writing
4022  *
4023  * Tunables may be removed or added as future performance improvements are
4024  * integrated, and also may become zpool properties.
4025  *
4026  * There are three key functions that control how the L2ARC warms up:
4027  *
4028  *      l2arc_write_eligible()  check if a buffer is eligible to cache
4029  *      l2arc_write_size()      calculate how much to write
4030  *      l2arc_write_interval()  calculate sleep delay between writes
4031  *
4032  * These three functions determine what to write, how much, and how quickly
4033  * to send writes.
4034  */
4035 
4036 static boolean_t
4037 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
4038 {
4039         /*
4040          * A buffer is *not* eligible for the L2ARC if it:
4041          * 1. belongs to a different spa.
4042          * 2. is already cached on the L2ARC.
4043          * 3. has an I/O in progress (it may be an incomplete read).
4044          * 4. is flagged not eligible (zfs property).
4045          */
4046         if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
4047             HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
4048                 return (B_FALSE);
4049 
4050         return (B_TRUE);
4051 }
4052 
4053 static uint64_t
4054 l2arc_write_size(void)
4055 {
4056         uint64_t size;
4057 
4058         /*
4059          * Make sure our globals have meaningful values in case the user
4060          * altered them.
4061          */
4062         size = l2arc_write_max;
4063         if (size == 0) {
4064                 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
4065                     "be greater than zero, resetting it to the default (%d)",
4066                     L2ARC_WRITE_SIZE);
4067                 size = l2arc_write_max = L2ARC_WRITE_SIZE;
4068         }
4069 
4070         if (arc_warm == B_FALSE)
4071                 size += l2arc_write_boost;
4072 
4073         return (size);
4074 
4075 }
4076 
4077 static clock_t
4078 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
4079 {
4080         clock_t interval, next, now;
4081 
4082         /*
4083          * If the ARC lists are busy, increase our write rate; if the
4084          * lists are stale, idle back.  This is achieved by checking
4085          * how much we previously wrote - if it was more than half of
4086          * what we wanted, schedule the next write much sooner.
4087          */
4088         if (l2arc_feed_again && wrote > (wanted / 2))
4089                 interval = (hz * l2arc_feed_min_ms) / 1000;
4090         else
4091                 interval = hz * l2arc_feed_secs;
4092 
4093         now = ddi_get_lbolt();
4094         next = MAX(now, MIN(now + interval, began + interval));
4095 
4096         return (next);
4097 }
4098 
4099 static void
4100 l2arc_hdr_stat_add(void)
4101 {
4102         ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
4103         ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
4104 }
4105 
4106 static void
4107 l2arc_hdr_stat_remove(void)
4108 {
4109         ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
4110         ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
4111 }
4112 
4113 /*
4114  * Cycle through L2ARC devices.  This is how L2ARC load balances.
4115  * If a device is returned, this also returns holding the spa config lock.
4116  */
4117 static l2arc_dev_t *
4118 l2arc_dev_get_next(void)
4119 {
4120         l2arc_dev_t *first, *next = NULL;
4121 
4122         /*
4123          * Lock out the removal of spas (spa_namespace_lock), then removal
4124          * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
4125          * both locks will be dropped and a spa config lock held instead.
4126          */
4127         mutex_enter(&spa_namespace_lock);
4128         mutex_enter(&l2arc_dev_mtx);
4129 
4130         /* if there are no vdevs, there is nothing to do */
4131         if (l2arc_ndev == 0)
4132                 goto out;
4133 
4134         first = NULL;
4135         next = l2arc_dev_last;
4136         do {
4137                 /* loop around the list looking for a non-faulted vdev */
4138                 if (next == NULL) {
4139                         next = list_head(l2arc_dev_list);
4140                 } else {
4141                         next = list_next(l2arc_dev_list, next);
4142                         if (next == NULL)
4143                                 next = list_head(l2arc_dev_list);
4144                 }
4145 
4146                 /* if we have come back to the start, bail out */
4147                 if (first == NULL)
4148                         first = next;
4149                 else if (next == first)
4150                         break;
4151 
4152         } while (vdev_is_dead(next->l2ad_vdev));
4153 
4154         /* if we were unable to find any usable vdevs, return NULL */
4155         if (vdev_is_dead(next->l2ad_vdev))
4156                 next = NULL;
4157 
4158         l2arc_dev_last = next;
4159 
4160 out:
4161         mutex_exit(&l2arc_dev_mtx);
4162 
4163         /*
4164          * Grab the config lock to prevent the 'next' device from being
4165          * removed while we are writing to it.
4166          */
4167         if (next != NULL)
4168                 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
4169         mutex_exit(&spa_namespace_lock);
4170 
4171         return (next);
4172 }
4173 
4174 /*
4175  * Free buffers that were tagged for destruction.
4176  */
4177 static void
4178 l2arc_do_free_on_write()
4179 {
4180         list_t *buflist;
4181         l2arc_data_free_t *df, *df_prev;
4182 
4183         mutex_enter(&l2arc_free_on_write_mtx);
4184         buflist = l2arc_free_on_write;
4185 
4186         for (df = list_tail(buflist); df; df = df_prev) {
4187                 df_prev = list_prev(buflist, df);
4188                 ASSERT(df->l2df_data != NULL);
4189                 ASSERT(df->l2df_func != NULL);
4190                 df->l2df_func(df->l2df_data, df->l2df_size);
4191                 list_remove(buflist, df);
4192                 kmem_free(df, sizeof (l2arc_data_free_t));
4193         }
4194 
4195         mutex_exit(&l2arc_free_on_write_mtx);
4196 }
4197 
4198 /*
4199  * A write to a cache device has completed.  Update all headers to allow
4200  * reads from these buffers to begin.
4201  */
4202 static void
4203 l2arc_write_done(zio_t *zio)
4204 {
4205         l2arc_write_callback_t *cb;
4206         l2arc_dev_t *dev;
4207         list_t *buflist;
4208         arc_buf_hdr_t *head, *ab, *ab_prev;
4209         l2arc_buf_hdr_t *abl2;
4210         kmutex_t *hash_lock;
4211 
4212         cb = zio->io_private;
4213         ASSERT(cb != NULL);
4214         dev = cb->l2wcb_dev;
4215         ASSERT(dev != NULL);
4216         head = cb->l2wcb_head;
4217         ASSERT(head != NULL);
4218         buflist = dev->l2ad_buflist;
4219         ASSERT(buflist != NULL);
4220         DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
4221             l2arc_write_callback_t *, cb);
4222 
4223         if (zio->io_error != 0)
4224                 ARCSTAT_BUMP(arcstat_l2_writes_error);
4225 
4226         mutex_enter(&l2arc_buflist_mtx);
4227 
4228         /*
4229          * All writes completed, or an error was hit.
4230          */
4231         for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
4232                 ab_prev = list_prev(buflist, ab);
4233 
4234                 hash_lock = HDR_LOCK(ab);
4235                 if (!mutex_tryenter(hash_lock)) {
4236                         /*
4237                          * This buffer misses out.  It may be in a stage
4238                          * of eviction.  Its ARC_L2_WRITING flag will be
4239                          * left set, denying reads to this buffer.
4240                          */
4241                         ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
4242                         continue;
4243                 }
4244 
4245                 abl2 = ab->b_l2hdr;
4246 
4247                 /*
4248                  * Release the temporary compressed buffer as soon as possible.
4249                  */
4250                 if (abl2->b_compress != ZIO_COMPRESS_OFF)
4251                         l2arc_release_cdata_buf(ab);
4252 
4253                 if (zio->io_error != 0) {
4254                         /*
4255                          * Error - drop L2ARC entry.
4256                          */
4257                         list_remove(buflist, ab);
4258                         ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4259                         ab->b_l2hdr = NULL;
4260                         kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4261                         ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4262                 }
4263 
4264                 /*
4265                  * Allow ARC to begin reads to this L2ARC entry.
4266                  */
4267                 ab->b_flags &= ~ARC_L2_WRITING;
4268 
4269                 mutex_exit(hash_lock);
4270         }
4271 
4272         atomic_inc_64(&l2arc_writes_done);
4273         list_remove(buflist, head);
4274         kmem_cache_free(hdr_cache, head);
4275         mutex_exit(&l2arc_buflist_mtx);
4276 
4277         l2arc_do_free_on_write();
4278 
4279         kmem_free(cb, sizeof (l2arc_write_callback_t));
4280 }
4281 
4282 /*
4283  * A read to a cache device completed.  Validate buffer contents before
4284  * handing over to the regular ARC routines.
4285  */
4286 static void
4287 l2arc_read_done(zio_t *zio)
4288 {
4289         l2arc_read_callback_t *cb;
4290         arc_buf_hdr_t *hdr;
4291         arc_buf_t *buf;
4292         kmutex_t *hash_lock;
4293         int equal;
4294 
4295         ASSERT(zio->io_vd != NULL);
4296         ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
4297 
4298         spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
4299 
4300         cb = zio->io_private;
4301         ASSERT(cb != NULL);
4302         buf = cb->l2rcb_buf;
4303         ASSERT(buf != NULL);
4304 
4305         hash_lock = HDR_LOCK(buf->b_hdr);
4306         mutex_enter(hash_lock);
4307         hdr = buf->b_hdr;
4308         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4309 
4310         /*
4311          * If the buffer was compressed, decompress it first.
4312          */
4313         if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
4314                 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
4315         ASSERT(zio->io_data != NULL);
4316 
4317         /*
4318          * Check this survived the L2ARC journey.
4319          */
4320         equal = arc_cksum_equal(buf);
4321         if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
4322                 mutex_exit(hash_lock);
4323                 zio->io_private = buf;
4324                 zio->io_bp_copy = cb->l2rcb_bp;   /* XXX fix in L2ARC 2.0 */
4325                 zio->io_bp = &zio->io_bp_copy;        /* XXX fix in L2ARC 2.0 */
4326                 arc_read_done(zio);
4327         } else {
4328                 mutex_exit(hash_lock);
4329                 /*
4330                  * Buffer didn't survive caching.  Increment stats and
4331                  * reissue to the original storage device.
4332                  */
4333                 if (zio->io_error != 0) {
4334                         ARCSTAT_BUMP(arcstat_l2_io_error);
4335                 } else {
4336                         zio->io_error = SET_ERROR(EIO);
4337                 }
4338                 if (!equal)
4339                         ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4340 
4341                 /*
4342                  * If there's no waiter, issue an async i/o to the primary
4343                  * storage now.  If there *is* a waiter, the caller must
4344                  * issue the i/o in a context where it's OK to block.
4345                  */
4346                 if (zio->io_waiter == NULL) {
4347                         zio_t *pio = zio_unique_parent(zio);
4348 
4349                         ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4350 
4351                         zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4352                             buf->b_data, zio->io_size, arc_read_done, buf,
4353                             zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4354                 }
4355         }
4356 
4357         kmem_free(cb, sizeof (l2arc_read_callback_t));
4358 }
4359 
4360 /*
4361  * This is the list priority from which the L2ARC will search for pages to
4362  * cache.  This is used within loops (0..3) to cycle through lists in the
4363  * desired order.  This order can have a significant effect on cache
4364  * performance.
4365  *
4366  * Currently the metadata lists are hit first, MFU then MRU, followed by
4367  * the data lists.  This function returns a locked list, and also returns
4368  * the lock pointer.
4369  */
4370 static list_t *
4371 l2arc_list_locked(int list_num, kmutex_t **lock)
4372 {
4373         list_t *list = NULL;
4374 
4375         ASSERT(list_num >= 0 && list_num <= 3);
4376 
4377         switch (list_num) {
4378         case 0:
4379                 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4380                 *lock = &arc_mfu->arcs_mtx;
4381                 break;
4382         case 1:
4383                 list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4384                 *lock = &arc_mru->arcs_mtx;
4385                 break;
4386         case 2:
4387                 list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4388                 *lock = &arc_mfu->arcs_mtx;
4389                 break;
4390         case 3:
4391                 list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4392                 *lock = &arc_mru->arcs_mtx;
4393                 break;
4394         }
4395 
4396         ASSERT(!(MUTEX_HELD(*lock)));
4397         mutex_enter(*lock);
4398         return (list);
4399 }
4400 
4401 /*
4402  * Evict buffers from the device write hand to the distance specified in
4403  * bytes.  This distance may span populated buffers, it may span nothing.
4404  * This is clearing a region on the L2ARC device ready for writing.
4405  * If the 'all' boolean is set, every buffer is evicted.
4406  */
4407 static void
4408 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4409 {
4410         list_t *buflist;
4411         l2arc_buf_hdr_t *abl2;
4412         arc_buf_hdr_t *ab, *ab_prev;
4413         kmutex_t *hash_lock;
4414         uint64_t taddr;
4415 
4416         buflist = dev->l2ad_buflist;
4417 
4418         if (buflist == NULL)
4419                 return;
4420 
4421         if (!all && dev->l2ad_first) {
4422                 /*
4423                  * This is the first sweep through the device.  There is
4424                  * nothing to evict.
4425                  */
4426                 return;
4427         }
4428 
4429         if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4430                 /*
4431                  * When nearing the end of the device, evict to the end
4432                  * before the device write hand jumps to the start.
4433                  */
4434                 taddr = dev->l2ad_end;
4435         } else {
4436                 taddr = dev->l2ad_hand + distance;
4437         }
4438         DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4439             uint64_t, taddr, boolean_t, all);
4440 
4441 top:
4442         mutex_enter(&l2arc_buflist_mtx);
4443         for (ab = list_tail(buflist); ab; ab = ab_prev) {
4444                 ab_prev = list_prev(buflist, ab);
4445 
4446                 hash_lock = HDR_LOCK(ab);
4447                 if (!mutex_tryenter(hash_lock)) {
4448                         /*
4449                          * Missed the hash lock.  Retry.
4450                          */
4451                         ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4452                         mutex_exit(&l2arc_buflist_mtx);
4453                         mutex_enter(hash_lock);
4454                         mutex_exit(hash_lock);
4455                         goto top;
4456                 }
4457 
4458                 if (HDR_L2_WRITE_HEAD(ab)) {
4459                         /*
4460                          * We hit a write head node.  Leave it for
4461                          * l2arc_write_done().
4462                          */
4463                         list_remove(buflist, ab);
4464                         mutex_exit(hash_lock);
4465                         continue;
4466                 }
4467 
4468                 if (!all && ab->b_l2hdr != NULL &&
4469                     (ab->b_l2hdr->b_daddr > taddr ||
4470                     ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4471                         /*
4472                          * We've evicted to the target address,
4473                          * or the end of the device.
4474                          */
4475                         mutex_exit(hash_lock);
4476                         break;
4477                 }
4478 
4479                 if (HDR_FREE_IN_PROGRESS(ab)) {
4480                         /*
4481                          * Already on the path to destruction.
4482                          */
4483                         mutex_exit(hash_lock);
4484                         continue;
4485                 }
4486 
4487                 if (ab->b_state == arc_l2c_only) {
4488                         ASSERT(!HDR_L2_READING(ab));
4489                         /*
4490                          * This doesn't exist in the ARC.  Destroy.
4491                          * arc_hdr_destroy() will call list_remove()
4492                          * and decrement arcstat_l2_size.
4493                          */
4494                         arc_change_state(arc_anon, ab, hash_lock);
4495                         arc_hdr_destroy(ab);
4496                 } else {
4497                         /*
4498                          * Invalidate issued or about to be issued
4499                          * reads, since we may be about to write
4500                          * over this location.
4501                          */
4502                         if (HDR_L2_READING(ab)) {
4503                                 ARCSTAT_BUMP(arcstat_l2_evict_reading);
4504                                 ab->b_flags |= ARC_L2_EVICTED;
4505                         }
4506 
4507                         /*
4508                          * Tell ARC this no longer exists in L2ARC.
4509                          */
4510                         if (ab->b_l2hdr != NULL) {
4511                                 abl2 = ab->b_l2hdr;
4512                                 ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4513                                 ab->b_l2hdr = NULL;
4514                                 kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4515                                 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4516                         }
4517                         list_remove(buflist, ab);
4518 
4519                         /*
4520                          * This may have been leftover after a
4521                          * failed write.
4522                          */
4523                         ab->b_flags &= ~ARC_L2_WRITING;
4524                 }
4525                 mutex_exit(hash_lock);
4526         }
4527         mutex_exit(&l2arc_buflist_mtx);
4528 
4529         vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4530         dev->l2ad_evict = taddr;
4531 }
4532 
4533 /*
4534  * Find and write ARC buffers to the L2ARC device.
4535  *
4536  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4537  * for reading until they have completed writing.
4538  * The headroom_boost is an in-out parameter used to maintain headroom boost
4539  * state between calls to this function.
4540  *
4541  * Returns the number of bytes actually written (which may be smaller than
4542  * the delta by which the device hand has changed due to alignment).
4543  */
4544 static uint64_t
4545 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
4546     boolean_t *headroom_boost)
4547 {
4548         arc_buf_hdr_t *ab, *ab_prev, *head;
4549         list_t *list;
4550         uint64_t write_asize, write_psize, write_sz, headroom,
4551             buf_compress_minsz;
4552         void *buf_data;
4553         kmutex_t *list_lock;
4554         boolean_t full;
4555         l2arc_write_callback_t *cb;
4556         zio_t *pio, *wzio;
4557         uint64_t guid = spa_load_guid(spa);
4558         const boolean_t do_headroom_boost = *headroom_boost;
4559 
4560         ASSERT(dev->l2ad_vdev != NULL);
4561 
4562         /* Lower the flag now, we might want to raise it again later. */
4563         *headroom_boost = B_FALSE;
4564 
4565         pio = NULL;
4566         write_sz = write_asize = write_psize = 0;
4567         full = B_FALSE;
4568         head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4569         head->b_flags |= ARC_L2_WRITE_HEAD;
4570 
4571         /*
4572          * We will want to try to compress buffers that are at least 2x the
4573          * device sector size.
4574          */
4575         buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
4576 
4577         /*
4578          * Copy buffers for L2ARC writing.
4579          */
4580         mutex_enter(&l2arc_buflist_mtx);
4581         for (int try = 0; try <= 3; try++) {
4582                 uint64_t passed_sz = 0;
4583 
4584                 list = l2arc_list_locked(try, &list_lock);
4585 
4586                 /*
4587                  * L2ARC fast warmup.
4588                  *
4589                  * Until the ARC is warm and starts to evict, read from the
4590                  * head of the ARC lists rather than the tail.
4591                  */
4592                 if (arc_warm == B_FALSE)
4593                         ab = list_head(list);
4594                 else
4595                         ab = list_tail(list);
4596 
4597                 headroom = target_sz * l2arc_headroom;
4598                 if (do_headroom_boost)
4599                         headroom = (headroom * l2arc_headroom_boost) / 100;
4600 
4601                 for (; ab; ab = ab_prev) {
4602                         l2arc_buf_hdr_t *l2hdr;
4603                         kmutex_t *hash_lock;
4604                         uint64_t buf_sz;
4605 
4606                         if (arc_warm == B_FALSE)
4607                                 ab_prev = list_next(list, ab);
4608                         else
4609                                 ab_prev = list_prev(list, ab);
4610 
4611                         hash_lock = HDR_LOCK(ab);
4612                         if (!mutex_tryenter(hash_lock)) {
4613                                 /*
4614                                  * Skip this buffer rather than waiting.
4615                                  */
4616                                 continue;
4617                         }
4618 
4619                         passed_sz += ab->b_size;
4620                         if (passed_sz > headroom) {
4621                                 /*
4622                                  * Searched too far.
4623                                  */
4624                                 mutex_exit(hash_lock);
4625                                 break;
4626                         }
4627 
4628                         if (!l2arc_write_eligible(guid, ab)) {
4629                                 mutex_exit(hash_lock);
4630                                 continue;
4631                         }
4632 
4633                         if ((write_sz + ab->b_size) > target_sz) {
4634                                 full = B_TRUE;
4635                                 mutex_exit(hash_lock);
4636                                 break;
4637                         }
4638 
4639                         if (pio == NULL) {
4640                                 /*
4641                                  * Insert a dummy header on the buflist so
4642                                  * l2arc_write_done() can find where the
4643                                  * write buffers begin without searching.
4644                                  */
4645                                 list_insert_head(dev->l2ad_buflist, head);
4646 
4647                                 cb = kmem_alloc(
4648                                     sizeof (l2arc_write_callback_t), KM_SLEEP);
4649                                 cb->l2wcb_dev = dev;
4650                                 cb->l2wcb_head = head;
4651                                 pio = zio_root(spa, l2arc_write_done, cb,
4652                                     ZIO_FLAG_CANFAIL);
4653                         }
4654 
4655                         /*
4656                          * Create and add a new L2ARC header.
4657                          */
4658                         l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4659                         l2hdr->b_dev = dev;
4660                         ab->b_flags |= ARC_L2_WRITING;
4661 
4662                         /*
4663                          * Temporarily stash the data buffer in b_tmp_cdata.
4664                          * The subsequent write step will pick it up from
4665                          * there. This is because can't access ab->b_buf
4666                          * without holding the hash_lock, which we in turn
4667                          * can't access without holding the ARC list locks
4668                          * (which we want to avoid during compression/writing).
4669                          */
4670                         l2hdr->b_compress = ZIO_COMPRESS_OFF;
4671                         l2hdr->b_asize = ab->b_size;
4672                         l2hdr->b_tmp_cdata = ab->b_buf->b_data;
4673 
4674                         buf_sz = ab->b_size;
4675                         ab->b_l2hdr = l2hdr;
4676 
4677                         list_insert_head(dev->l2ad_buflist, ab);
4678 
4679                         /*
4680                          * Compute and store the buffer cksum before
4681                          * writing.  On debug the cksum is verified first.
4682                          */
4683                         arc_cksum_verify(ab->b_buf);
4684                         arc_cksum_compute(ab->b_buf, B_TRUE);
4685 
4686                         mutex_exit(hash_lock);
4687 
4688                         write_sz += buf_sz;
4689                 }
4690 
4691                 mutex_exit(list_lock);
4692 
4693                 if (full == B_TRUE)
4694                         break;
4695         }
4696 
4697         /* No buffers selected for writing? */
4698         if (pio == NULL) {
4699                 ASSERT0(write_sz);
4700                 mutex_exit(&l2arc_buflist_mtx);
4701                 kmem_cache_free(hdr_cache, head);
4702                 return (0);
4703         }
4704 
4705         /*
4706          * Now start writing the buffers. We're starting at the write head
4707          * and work backwards, retracing the course of the buffer selector
4708          * loop above.
4709          */
4710         for (ab = list_prev(dev->l2ad_buflist, head); ab;
4711             ab = list_prev(dev->l2ad_buflist, ab)) {
4712                 l2arc_buf_hdr_t *l2hdr;
4713                 uint64_t buf_sz;
4714 
4715                 /*
4716                  * We shouldn't need to lock the buffer here, since we flagged
4717                  * it as ARC_L2_WRITING in the previous step, but we must take
4718                  * care to only access its L2 cache parameters. In particular,
4719                  * ab->b_buf may be invalid by now due to ARC eviction.
4720                  */
4721                 l2hdr = ab->b_l2hdr;
4722                 l2hdr->b_daddr = dev->l2ad_hand;
4723 
4724                 if ((ab->b_flags & ARC_L2COMPRESS) &&
4725                     l2hdr->b_asize >= buf_compress_minsz) {
4726                         if (l2arc_compress_buf(l2hdr)) {
4727                                 /*
4728                                  * If compression succeeded, enable headroom
4729                                  * boost on the next scan cycle.
4730                                  */
4731                                 *headroom_boost = B_TRUE;
4732                         }
4733                 }
4734 
4735                 /*
4736                  * Pick up the buffer data we had previously stashed away
4737                  * (and now potentially also compressed).
4738                  */
4739                 buf_data = l2hdr->b_tmp_cdata;
4740                 buf_sz = l2hdr->b_asize;
4741 
4742                 /* Compression may have squashed the buffer to zero length. */
4743                 if (buf_sz != 0) {
4744                         uint64_t buf_p_sz;
4745 
4746                         wzio = zio_write_phys(pio, dev->l2ad_vdev,
4747                             dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4748                             NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4749                             ZIO_FLAG_CANFAIL, B_FALSE);
4750 
4751                         DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4752                             zio_t *, wzio);
4753                         (void) zio_nowait(wzio);
4754 
4755                         write_asize += buf_sz;
4756                         /*
4757                          * Keep the clock hand suitably device-aligned.
4758                          */
4759                         buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4760                         write_psize += buf_p_sz;
4761                         dev->l2ad_hand += buf_p_sz;
4762                 }
4763         }
4764 
4765         mutex_exit(&l2arc_buflist_mtx);
4766 
4767         ASSERT3U(write_asize, <=, target_sz);
4768         ARCSTAT_BUMP(arcstat_l2_writes_sent);
4769         ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
4770         ARCSTAT_INCR(arcstat_l2_size, write_sz);
4771         ARCSTAT_INCR(arcstat_l2_asize, write_asize);
4772         vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
4773 
4774         /*
4775          * Bump device hand to the device start if it is approaching the end.
4776          * l2arc_evict() will already have evicted ahead for this case.
4777          */
4778         if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4779                 vdev_space_update(dev->l2ad_vdev,
4780                     dev->l2ad_end - dev->l2ad_hand, 0, 0);
4781                 dev->l2ad_hand = dev->l2ad_start;
4782                 dev->l2ad_evict = dev->l2ad_start;
4783                 dev->l2ad_first = B_FALSE;
4784         }
4785 
4786         dev->l2ad_writing = B_TRUE;
4787         (void) zio_wait(pio);
4788         dev->l2ad_writing = B_FALSE;
4789 
4790         return (write_asize);
4791 }
4792 
4793 /*
4794  * Compresses an L2ARC buffer.
4795  * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its
4796  * size in l2hdr->b_asize. This routine tries to compress the data and
4797  * depending on the compression result there are three possible outcomes:
4798  * *) The buffer was incompressible. The original l2hdr contents were left
4799  *    untouched and are ready for writing to an L2 device.
4800  * *) The buffer was all-zeros, so there is no need to write it to an L2
4801  *    device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
4802  *    set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
4803  * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
4804  *    data buffer which holds the compressed data to be written, and b_asize
4805  *    tells us how much data there is. b_compress is set to the appropriate
4806  *    compression algorithm. Once writing is done, invoke
4807  *    l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
4808  *
4809  * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
4810  * buffer was incompressible).
4811  */
4812 static boolean_t
4813 l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
4814 {
4815         void *cdata;
4816         size_t csize, len;
4817 
4818         ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF);
4819         ASSERT(l2hdr->b_tmp_cdata != NULL);
4820 
4821         len = l2hdr->b_asize;
4822         cdata = zio_data_buf_alloc(len);
4823         csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata,
4824             cdata, l2hdr->b_asize);
4825 
4826         if (csize == 0) {
4827                 /* zero block, indicate that there's nothing to write */
4828                 zio_data_buf_free(cdata, len);
4829                 l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
4830                 l2hdr->b_asize = 0;
4831                 l2hdr->b_tmp_cdata = NULL;
4832                 ARCSTAT_BUMP(arcstat_l2_compress_zeros);
4833                 return (B_TRUE);
4834         } else if (csize > 0 && csize < len) {
4835                 /*
4836                  * Compression succeeded, we'll keep the cdata around for
4837                  * writing and release it afterwards.
4838                  */
4839                 l2hdr->b_compress = ZIO_COMPRESS_LZ4;
4840                 l2hdr->b_asize = csize;
4841                 l2hdr->b_tmp_cdata = cdata;
4842                 ARCSTAT_BUMP(arcstat_l2_compress_successes);
4843                 return (B_TRUE);
4844         } else {
4845                 /*
4846                  * Compression failed, release the compressed buffer.
4847                  * l2hdr will be left unmodified.
4848                  */
4849                 zio_data_buf_free(cdata, len);
4850                 ARCSTAT_BUMP(arcstat_l2_compress_failures);
4851                 return (B_FALSE);
4852         }
4853 }
4854 
4855 /*
4856  * Decompresses a zio read back from an l2arc device. On success, the
4857  * underlying zio's io_data buffer is overwritten by the uncompressed
4858  * version. On decompression error (corrupt compressed stream), the
4859  * zio->io_error value is set to signal an I/O error.
4860  *
4861  * Please note that the compressed data stream is not checksummed, so
4862  * if the underlying device is experiencing data corruption, we may feed
4863  * corrupt data to the decompressor, so the decompressor needs to be
4864  * able to handle this situation (LZ4 does).
4865  */
4866 static void
4867 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
4868 {
4869         ASSERT(L2ARC_IS_VALID_COMPRESS(c));
4870 
4871         if (zio->io_error != 0) {
4872                 /*
4873                  * An io error has occured, just restore the original io
4874                  * size in preparation for a main pool read.
4875                  */
4876                 zio->io_orig_size = zio->io_size = hdr->b_size;
4877                 return;
4878         }
4879 
4880         if (c == ZIO_COMPRESS_EMPTY) {
4881                 /*
4882                  * An empty buffer results in a null zio, which means we
4883                  * need to fill its io_data after we're done restoring the
4884                  * buffer's contents.
4885                  */
4886                 ASSERT(hdr->b_buf != NULL);
4887                 bzero(hdr->b_buf->b_data, hdr->b_size);
4888                 zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
4889         } else {
4890                 ASSERT(zio->io_data != NULL);
4891                 /*
4892                  * We copy the compressed data from the start of the arc buffer
4893                  * (the zio_read will have pulled in only what we need, the
4894                  * rest is garbage which we will overwrite at decompression)
4895                  * and then decompress back to the ARC data buffer. This way we
4896                  * can minimize copying by simply decompressing back over the
4897                  * original compressed data (rather than decompressing to an
4898                  * aux buffer and then copying back the uncompressed buffer,
4899                  * which is likely to be much larger).
4900                  */
4901                 uint64_t csize;
4902                 void *cdata;
4903 
4904                 csize = zio->io_size;
4905                 cdata = zio_data_buf_alloc(csize);
4906                 bcopy(zio->io_data, cdata, csize);
4907                 if (zio_decompress_data(c, cdata, zio->io_data, csize,
4908                     hdr->b_size) != 0)
4909                         zio->io_error = EIO;
4910                 zio_data_buf_free(cdata, csize);
4911         }
4912 
4913         /* Restore the expected uncompressed IO size. */
4914         zio->io_orig_size = zio->io_size = hdr->b_size;
4915 }
4916 
4917 /*
4918  * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
4919  * This buffer serves as a temporary holder of compressed data while
4920  * the buffer entry is being written to an l2arc device. Once that is
4921  * done, we can dispose of it.
4922  */
4923 static void
4924 l2arc_release_cdata_buf(arc_buf_hdr_t *ab)
4925 {
4926         l2arc_buf_hdr_t *l2hdr = ab->b_l2hdr;
4927 
4928         if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
4929                 /*
4930                  * If the data was compressed, then we've allocated a
4931                  * temporary buffer for it, so now we need to release it.
4932                  */
4933                 ASSERT(l2hdr->b_tmp_cdata != NULL);
4934                 zio_data_buf_free(l2hdr->b_tmp_cdata, ab->b_size);
4935         }
4936         l2hdr->b_tmp_cdata = NULL;
4937 }
4938 
4939 /*
4940  * This thread feeds the L2ARC at regular intervals.  This is the beating
4941  * heart of the L2ARC.
4942  */
4943 static void
4944 l2arc_feed_thread(void)
4945 {
4946         callb_cpr_t cpr;
4947         l2arc_dev_t *dev;
4948         spa_t *spa;
4949         uint64_t size, wrote;
4950         clock_t begin, next = ddi_get_lbolt();
4951         boolean_t headroom_boost = B_FALSE;
4952 
4953         CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4954 
4955         mutex_enter(&l2arc_feed_thr_lock);
4956 
4957         while (l2arc_thread_exit == 0) {
4958                 CALLB_CPR_SAFE_BEGIN(&cpr);
4959                 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
4960                     next);
4961                 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4962                 next = ddi_get_lbolt() + hz;
4963 
4964                 /*
4965                  * Quick check for L2ARC devices.
4966                  */
4967                 mutex_enter(&l2arc_dev_mtx);
4968                 if (l2arc_ndev == 0) {
4969                         mutex_exit(&l2arc_dev_mtx);
4970                         continue;
4971                 }
4972                 mutex_exit(&l2arc_dev_mtx);
4973                 begin = ddi_get_lbolt();
4974 
4975                 /*
4976                  * This selects the next l2arc device to write to, and in
4977                  * doing so the next spa to feed from: dev->l2ad_spa.   This
4978                  * will return NULL if there are now no l2arc devices or if
4979                  * they are all faulted.
4980                  *
4981                  * If a device is returned, its spa's config lock is also
4982                  * held to prevent device removal.  l2arc_dev_get_next()
4983                  * will grab and release l2arc_dev_mtx.
4984                  */
4985                 if ((dev = l2arc_dev_get_next()) == NULL)
4986                         continue;
4987 
4988                 spa = dev->l2ad_spa;
4989                 ASSERT(spa != NULL);
4990 
4991                 /*
4992                  * If the pool is read-only then force the feed thread to
4993                  * sleep a little longer.
4994                  */
4995                 if (!spa_writeable(spa)) {
4996                         next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
4997                         spa_config_exit(spa, SCL_L2ARC, dev);
4998                         continue;
4999                 }
5000 
5001                 /*
5002                  * Avoid contributing to memory pressure.
5003                  */
5004                 if (arc_reclaim_needed()) {
5005                         ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
5006                         spa_config_exit(spa, SCL_L2ARC, dev);
5007                         continue;
5008                 }
5009 
5010                 ARCSTAT_BUMP(arcstat_l2_feeds);
5011 
5012                 size = l2arc_write_size();
5013 
5014                 /*
5015                  * Evict L2ARC buffers that will be overwritten.
5016                  */
5017                 l2arc_evict(dev, size, B_FALSE);
5018 
5019                 /*
5020                  * Write ARC buffers.
5021                  */
5022                 wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
5023 
5024                 /*
5025                  * Calculate interval between writes.
5026                  */
5027                 next = l2arc_write_interval(begin, size, wrote);
5028                 spa_config_exit(spa, SCL_L2ARC, dev);
5029         }
5030 
5031         l2arc_thread_exit = 0;
5032         cv_broadcast(&l2arc_feed_thr_cv);
5033         CALLB_CPR_EXIT(&cpr);               /* drops l2arc_feed_thr_lock */
5034         thread_exit();
5035 }
5036 
5037 boolean_t
5038 l2arc_vdev_present(vdev_t *vd)
5039 {
5040         l2arc_dev_t *dev;
5041 
5042         mutex_enter(&l2arc_dev_mtx);
5043         for (dev = list_head(l2arc_dev_list); dev != NULL;
5044             dev = list_next(l2arc_dev_list, dev)) {
5045                 if (dev->l2ad_vdev == vd)
5046                         break;
5047         }
5048         mutex_exit(&l2arc_dev_mtx);
5049 
5050         return (dev != NULL);
5051 }
5052 
5053 /*
5054  * Add a vdev for use by the L2ARC.  By this point the spa has already
5055  * validated the vdev and opened it.
5056  */
5057 void
5058 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
5059 {
5060         l2arc_dev_t *adddev;
5061 
5062         ASSERT(!l2arc_vdev_present(vd));
5063 
5064         /*
5065          * Create a new l2arc device entry.
5066          */
5067         adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
5068         adddev->l2ad_spa = spa;
5069         adddev->l2ad_vdev = vd;
5070         adddev->l2ad_start = VDEV_LABEL_START_SIZE;
5071         adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
5072         adddev->l2ad_hand = adddev->l2ad_start;
5073         adddev->l2ad_evict = adddev->l2ad_start;
5074         adddev->l2ad_first = B_TRUE;
5075         adddev->l2ad_writing = B_FALSE;
5076 
5077         /*
5078          * This is a list of all ARC buffers that are still valid on the
5079          * device.
5080          */
5081         adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
5082         list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
5083             offsetof(arc_buf_hdr_t, b_l2node));
5084 
5085         vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
5086 
5087         /*
5088          * Add device to global list
5089          */
5090         mutex_enter(&l2arc_dev_mtx);
5091         list_insert_head(l2arc_dev_list, adddev);
5092         atomic_inc_64(&l2arc_ndev);
5093         mutex_exit(&l2arc_dev_mtx);
5094 }
5095 
5096 /*
5097  * Remove a vdev from the L2ARC.
5098  */
5099 void
5100 l2arc_remove_vdev(vdev_t *vd)
5101 {
5102         l2arc_dev_t *dev, *nextdev, *remdev = NULL;
5103 
5104         /*
5105          * Find the device by vdev
5106          */
5107         mutex_enter(&l2arc_dev_mtx);
5108         for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
5109                 nextdev = list_next(l2arc_dev_list, dev);
5110                 if (vd == dev->l2ad_vdev) {
5111                         remdev = dev;
5112                         break;
5113                 }
5114         }
5115         ASSERT(remdev != NULL);
5116 
5117         /*
5118          * Remove device from global list
5119          */
5120         list_remove(l2arc_dev_list, remdev);
5121         l2arc_dev_last = NULL;          /* may have been invalidated */
5122         atomic_dec_64(&l2arc_ndev);
5123         mutex_exit(&l2arc_dev_mtx);
5124 
5125         /*
5126          * Clear all buflists and ARC references.  L2ARC device flush.
5127          */
5128         l2arc_evict(remdev, 0, B_TRUE);
5129         list_destroy(remdev->l2ad_buflist);
5130         kmem_free(remdev->l2ad_buflist, sizeof (list_t));
5131         kmem_free(remdev, sizeof (l2arc_dev_t));
5132 }
5133 
5134 void
5135 l2arc_init(void)
5136 {
5137         l2arc_thread_exit = 0;
5138         l2arc_ndev = 0;
5139         l2arc_writes_sent = 0;
5140         l2arc_writes_done = 0;
5141 
5142         mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
5143         cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
5144         mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
5145         mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
5146         mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
5147 
5148         l2arc_dev_list = &L2ARC_dev_list;
5149         l2arc_free_on_write = &L2ARC_free_on_write;
5150         list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
5151             offsetof(l2arc_dev_t, l2ad_node));
5152         list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
5153             offsetof(l2arc_data_free_t, l2df_list_node));
5154 }
5155 
5156 void
5157 l2arc_fini(void)
5158 {
5159         /*
5160          * This is called from dmu_fini(), which is called from spa_fini();
5161          * Because of this, we can assume that all l2arc devices have
5162          * already been removed when the pools themselves were removed.
5163          */
5164 
5165         l2arc_do_free_on_write();
5166 
5167         mutex_destroy(&l2arc_feed_thr_lock);
5168         cv_destroy(&l2arc_feed_thr_cv);
5169         mutex_destroy(&l2arc_dev_mtx);
5170         mutex_destroy(&l2arc_buflist_mtx);
5171         mutex_destroy(&l2arc_free_on_write_mtx);
5172 
5173         list_destroy(l2arc_dev_list);
5174         list_destroy(l2arc_free_on_write);
5175 }
5176 
5177 void
5178 l2arc_start(void)
5179 {
5180         if (!(spa_mode_global & FWRITE))
5181                 return;
5182 
5183         (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
5184             TS_RUN, minclsyspri);
5185 }
5186 
5187 void
5188 l2arc_stop(void)
5189 {
5190         if (!(spa_mode_global & FWRITE))
5191                 return;
5192 
5193         mutex_enter(&l2arc_feed_thr_lock);
5194         cv_signal(&l2arc_feed_thr_cv);      /* kick thread out of startup */
5195         l2arc_thread_exit = 1;
5196         while (l2arc_thread_exit != 0)
5197                 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
5198         mutex_exit(&l2arc_feed_thr_lock);
5199 }