1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  24  * Copyright (c) 2013 by Delphix. All rights reserved.
  25  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  26  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  27  */
  28 
  29 /*
  30  * DVA-based Adjustable Replacement Cache
  31  *
  32  * While much of the theory of operation used here is
  33  * based on the self-tuning, low overhead replacement cache
  34  * presented by Megiddo and Modha at FAST 2003, there are some
  35  * significant differences:
  36  *
  37  * 1. The Megiddo and Modha model assumes any page is evictable.
  38  * Pages in its cache cannot be "locked" into memory.  This makes
  39  * the eviction algorithm simple: evict the last page in the list.
  40  * This also make the performance characteristics easy to reason
  41  * about.  Our cache is not so simple.  At any given moment, some
  42  * subset of the blocks in the cache are un-evictable because we
  43  * have handed out a reference to them.  Blocks are only evictable
  44  * when there are no external references active.  This makes
  45  * eviction far more problematic:  we choose to evict the evictable
  46  * blocks that are the "lowest" in the list.
  47  *
  48  * There are times when it is not possible to evict the requested
  49  * space.  In these circumstances we are unable to adjust the cache
  50  * size.  To prevent the cache growing unbounded at these times we
  51  * implement a "cache throttle" that slows the flow of new data
  52  * into the cache until we can make space available.
  53  *
  54  * 2. The Megiddo and Modha model assumes a fixed cache size.
  55  * Pages are evicted when the cache is full and there is a cache
  56  * miss.  Our model has a variable sized cache.  It grows with
  57  * high use, but also tries to react to memory pressure from the
  58  * operating system: decreasing its size when system memory is
  59  * tight.
  60  *
  61  * 3. The Megiddo and Modha model assumes a fixed page size. All
  62  * elements of the cache are therefore exactly the same size.  So
  63  * when adjusting the cache size following a cache miss, its simply
  64  * a matter of choosing a single page to evict.  In our model, we
  65  * have variable sized cache blocks (rangeing from 512 bytes to
  66  * 128K bytes).  We therefore choose a set of blocks to evict to make
  67  * space for a cache miss that approximates as closely as possible
  68  * the space used by the new block.
  69  *
  70  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
  71  * by N. Megiddo & D. Modha, FAST 2003
  72  */
  73 
  74 /*
  75  * The locking model:
  76  *
  77  * A new reference to a cache buffer can be obtained in two
  78  * ways: 1) via a hash table lookup using the DVA as a key,
  79  * or 2) via one of the ARC lists.  The arc_read() interface
  80  * uses method 1, while the internal arc algorithms for
  81  * adjusting the cache use method 2.  We therefore provide two
  82  * types of locks: 1) the hash table lock array, and 2) the
  83  * arc list locks.
  84  *
  85  * Buffers do not have their own mutexes, rather they rely on the
  86  * hash table mutexes for the bulk of their protection (i.e. most
  87  * fields in the arc_buf_hdr_t are protected by these mutexes).
  88  *
  89  * buf_hash_find() returns the appropriate mutex (held) when it
  90  * locates the requested buffer in the hash table.  It returns
  91  * NULL for the mutex if the buffer was not in the table.
  92  *
  93  * buf_hash_remove() expects the appropriate hash mutex to be
  94  * already held before it is invoked.
  95  *
  96  * Each arc state also has a mutex which is used to protect the
  97  * buffer list associated with the state.  When attempting to
  98  * obtain a hash table lock while holding an arc list lock you
  99  * must use: mutex_tryenter() to avoid deadlock.  Also note that
 100  * the active state mutex must be held before the ghost state mutex.
 101  *
 102  * Arc buffers may have an associated eviction callback function.
 103  * This function will be invoked prior to removing the buffer (e.g.
 104  * in arc_do_user_evicts()).  Note however that the data associated
 105  * with the buffer may be evicted prior to the callback.  The callback
 106  * must be made with *no locks held* (to prevent deadlock).  Additionally,
 107  * the users of callbacks must ensure that their private data is
 108  * protected from simultaneous callbacks from arc_buf_evict()
 109  * and arc_do_user_evicts().
 110  *
 111  * Note that the majority of the performance stats are manipulated
 112  * with atomic operations.
 113  *
 114  * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
 115  *
 116  *      - L2ARC buflist creation
 117  *      - L2ARC buflist eviction
 118  *      - L2ARC write completion, which walks L2ARC buflists
 119  *      - ARC header destruction, as it removes from L2ARC buflists
 120  *      - ARC header release, as it removes from L2ARC buflists
 121  */
 122 
 123 #include <sys/spa.h>
 124 #include <sys/zio.h>
 125 #include <sys/zio_compress.h>
 126 #include <sys/zfs_context.h>
 127 #include <sys/arc.h>
 128 #include <sys/refcount.h>
 129 #include <sys/vdev.h>
 130 #include <sys/vdev_impl.h>
 131 #include <sys/dsl_pool.h>
 132 #ifdef _KERNEL
 133 #include <sys/vmsystm.h>
 134 #include <vm/anon.h>
 135 #include <sys/fs/swapnode.h>
 136 #include <sys/dnlc.h>
 137 #endif
 138 #include <sys/callb.h>
 139 #include <sys/kstat.h>
 140 #include <zfs_fletcher.h>
 141 
 142 #ifndef _KERNEL
 143 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
 144 boolean_t arc_watch = B_FALSE;
 145 int arc_procfd;
 146 #endif
 147 
 148 static kmutex_t         arc_reclaim_thr_lock;
 149 static kcondvar_t       arc_reclaim_thr_cv;     /* used to signal reclaim thr */
 150 static uint8_t          arc_thread_exit;
 151 
 152 #define ARC_REDUCE_DNLC_PERCENT 3
 153 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
 154 
 155 typedef enum arc_reclaim_strategy {
 156         ARC_RECLAIM_AGGR,               /* Aggressive reclaim strategy */
 157         ARC_RECLAIM_CONS                /* Conservative reclaim strategy */
 158 } arc_reclaim_strategy_t;
 159 
 160 /*
 161  * The number of iterations through arc_evict_*() before we
 162  * drop & reacquire the lock.
 163  */
 164 int arc_evict_iterations = 100;
 165 
 166 /* number of seconds before growing cache again */
 167 static int              arc_grow_retry = 60;
 168 
 169 /* shift of arc_c for calculating both min and max arc_p */
 170 static int              arc_p_min_shift = 4;
 171 
 172 /* log2(fraction of arc to reclaim) */
 173 static int              arc_shrink_shift = 5;
 174 
 175 /*
 176  * minimum lifespan of a prefetch block in clock ticks
 177  * (initialized in arc_init())
 178  */
 179 static int              arc_min_prefetch_lifespan;
 180 
 181 /*
 182  * If this percent of memory is free, don't throttle.
 183  */
 184 int arc_lotsfree_percent = 10;
 185 
 186 static int arc_dead;
 187 
 188 /*
 189  * The arc has filled available memory and has now warmed up.
 190  */
 191 static boolean_t arc_warm;
 192 
 193 /*
 194  * These tunables are for performance analysis.
 195  */
 196 uint64_t zfs_arc_max;
 197 uint64_t zfs_arc_min;
 198 uint64_t zfs_arc_meta_limit = 0;
 199 int zfs_arc_grow_retry = 0;
 200 int zfs_arc_shrink_shift = 0;
 201 int zfs_arc_p_min_shift = 0;
 202 int zfs_disable_dup_eviction = 0;
 203 
 204 /*
 205  * Note that buffers can be in one of 6 states:
 206  *      ARC_anon        - anonymous (discussed below)
 207  *      ARC_mru         - recently used, currently cached
 208  *      ARC_mru_ghost   - recentely used, no longer in cache
 209  *      ARC_mfu         - frequently used, currently cached
 210  *      ARC_mfu_ghost   - frequently used, no longer in cache
 211  *      ARC_l2c_only    - exists in L2ARC but not other states
 212  * When there are no active references to the buffer, they are
 213  * are linked onto a list in one of these arc states.  These are
 214  * the only buffers that can be evicted or deleted.  Within each
 215  * state there are multiple lists, one for meta-data and one for
 216  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
 217  * etc.) is tracked separately so that it can be managed more
 218  * explicitly: favored over data, limited explicitly.
 219  *
 220  * Anonymous buffers are buffers that are not associated with
 221  * a DVA.  These are buffers that hold dirty block copies
 222  * before they are written to stable storage.  By definition,
 223  * they are "ref'd" and are considered part of arc_mru
 224  * that cannot be freed.  Generally, they will aquire a DVA
 225  * as they are written and migrate onto the arc_mru list.
 226  *
 227  * The ARC_l2c_only state is for buffers that are in the second
 228  * level ARC but no longer in any of the ARC_m* lists.  The second
 229  * level ARC itself may also contain buffers that are in any of
 230  * the ARC_m* states - meaning that a buffer can exist in two
 231  * places.  The reason for the ARC_l2c_only state is to keep the
 232  * buffer header in the hash table, so that reads that hit the
 233  * second level ARC benefit from these fast lookups.
 234  */
 235 
 236 typedef struct arc_state {
 237         list_t  arcs_list[ARC_BUFC_NUMTYPES];   /* list of evictable buffers */
 238         uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */
 239         uint64_t arcs_size;     /* total amount of data in this state */
 240         kmutex_t arcs_mtx;
 241 } arc_state_t;
 242 
 243 /* The 6 states: */
 244 static arc_state_t ARC_anon;
 245 static arc_state_t ARC_mru;
 246 static arc_state_t ARC_mru_ghost;
 247 static arc_state_t ARC_mfu;
 248 static arc_state_t ARC_mfu_ghost;
 249 static arc_state_t ARC_l2c_only;
 250 
 251 typedef struct arc_stats {
 252         kstat_named_t arcstat_hits;
 253         kstat_named_t arcstat_misses;
 254         kstat_named_t arcstat_demand_data_hits;
 255         kstat_named_t arcstat_demand_data_misses;
 256         kstat_named_t arcstat_demand_metadata_hits;
 257         kstat_named_t arcstat_demand_metadata_misses;
 258         kstat_named_t arcstat_prefetch_data_hits;
 259         kstat_named_t arcstat_prefetch_data_misses;
 260         kstat_named_t arcstat_prefetch_metadata_hits;
 261         kstat_named_t arcstat_prefetch_metadata_misses;
 262         kstat_named_t arcstat_mru_hits;
 263         kstat_named_t arcstat_mru_ghost_hits;
 264         kstat_named_t arcstat_mfu_hits;
 265         kstat_named_t arcstat_mfu_ghost_hits;
 266         kstat_named_t arcstat_deleted;
 267         kstat_named_t arcstat_recycle_miss;
 268         /*
 269          * Number of buffers that could not be evicted because the hash lock
 270          * was held by another thread.  The lock may not necessarily be held
 271          * by something using the same buffer, since hash locks are shared
 272          * by multiple buffers.
 273          */
 274         kstat_named_t arcstat_mutex_miss;
 275         /*
 276          * Number of buffers skipped because they have I/O in progress, are
 277          * indrect prefetch buffers that have not lived long enough, or are
 278          * not from the spa we're trying to evict from.
 279          */
 280         kstat_named_t arcstat_evict_skip;
 281         kstat_named_t arcstat_evict_l2_cached;
 282         kstat_named_t arcstat_evict_l2_eligible;
 283         kstat_named_t arcstat_evict_l2_ineligible;
 284         kstat_named_t arcstat_hash_elements;
 285         kstat_named_t arcstat_hash_elements_max;
 286         kstat_named_t arcstat_hash_collisions;
 287         kstat_named_t arcstat_hash_chains;
 288         kstat_named_t arcstat_hash_chain_max;
 289         kstat_named_t arcstat_p;
 290         kstat_named_t arcstat_c;
 291         kstat_named_t arcstat_c_min;
 292         kstat_named_t arcstat_c_max;
 293         kstat_named_t arcstat_size;
 294         kstat_named_t arcstat_hdr_size;
 295         kstat_named_t arcstat_data_size;
 296         kstat_named_t arcstat_other_size;
 297         kstat_named_t arcstat_l2_hits;
 298         kstat_named_t arcstat_l2_misses;
 299         kstat_named_t arcstat_l2_feeds;
 300         kstat_named_t arcstat_l2_rw_clash;
 301         kstat_named_t arcstat_l2_read_bytes;
 302         kstat_named_t arcstat_l2_write_bytes;
 303         kstat_named_t arcstat_l2_writes_sent;
 304         kstat_named_t arcstat_l2_writes_done;
 305         kstat_named_t arcstat_l2_writes_error;
 306         kstat_named_t arcstat_l2_writes_hdr_miss;
 307         kstat_named_t arcstat_l2_evict_lock_retry;
 308         kstat_named_t arcstat_l2_evict_reading;
 309         kstat_named_t arcstat_l2_free_on_write;
 310         kstat_named_t arcstat_l2_abort_lowmem;
 311         kstat_named_t arcstat_l2_cksum_bad;
 312         kstat_named_t arcstat_l2_io_error;
 313         kstat_named_t arcstat_l2_size;
 314         kstat_named_t arcstat_l2_asize;
 315         kstat_named_t arcstat_l2_hdr_size;
 316         kstat_named_t arcstat_l2_compress_successes;
 317         kstat_named_t arcstat_l2_compress_zeros;
 318         kstat_named_t arcstat_l2_compress_failures;
 319         kstat_named_t arcstat_memory_throttle_count;
 320         kstat_named_t arcstat_duplicate_buffers;
 321         kstat_named_t arcstat_duplicate_buffers_size;
 322         kstat_named_t arcstat_duplicate_reads;
 323         kstat_named_t arcstat_meta_used;
 324         kstat_named_t arcstat_meta_limit;
 325         kstat_named_t arcstat_meta_max;
 326 } arc_stats_t;
 327 
 328 static arc_stats_t arc_stats = {
 329         { "hits",                       KSTAT_DATA_UINT64 },
 330         { "misses",                     KSTAT_DATA_UINT64 },
 331         { "demand_data_hits",           KSTAT_DATA_UINT64 },
 332         { "demand_data_misses",         KSTAT_DATA_UINT64 },
 333         { "demand_metadata_hits",       KSTAT_DATA_UINT64 },
 334         { "demand_metadata_misses",     KSTAT_DATA_UINT64 },
 335         { "prefetch_data_hits",         KSTAT_DATA_UINT64 },
 336         { "prefetch_data_misses",       KSTAT_DATA_UINT64 },
 337         { "prefetch_metadata_hits",     KSTAT_DATA_UINT64 },
 338         { "prefetch_metadata_misses",   KSTAT_DATA_UINT64 },
 339         { "mru_hits",                   KSTAT_DATA_UINT64 },
 340         { "mru_ghost_hits",             KSTAT_DATA_UINT64 },
 341         { "mfu_hits",                   KSTAT_DATA_UINT64 },
 342         { "mfu_ghost_hits",             KSTAT_DATA_UINT64 },
 343         { "deleted",                    KSTAT_DATA_UINT64 },
 344         { "recycle_miss",               KSTAT_DATA_UINT64 },
 345         { "mutex_miss",                 KSTAT_DATA_UINT64 },
 346         { "evict_skip",                 KSTAT_DATA_UINT64 },
 347         { "evict_l2_cached",            KSTAT_DATA_UINT64 },
 348         { "evict_l2_eligible",          KSTAT_DATA_UINT64 },
 349         { "evict_l2_ineligible",        KSTAT_DATA_UINT64 },
 350         { "hash_elements",              KSTAT_DATA_UINT64 },
 351         { "hash_elements_max",          KSTAT_DATA_UINT64 },
 352         { "hash_collisions",            KSTAT_DATA_UINT64 },
 353         { "hash_chains",                KSTAT_DATA_UINT64 },
 354         { "hash_chain_max",             KSTAT_DATA_UINT64 },
 355         { "p",                          KSTAT_DATA_UINT64 },
 356         { "c",                          KSTAT_DATA_UINT64 },
 357         { "c_min",                      KSTAT_DATA_UINT64 },
 358         { "c_max",                      KSTAT_DATA_UINT64 },
 359         { "size",                       KSTAT_DATA_UINT64 },
 360         { "hdr_size",                   KSTAT_DATA_UINT64 },
 361         { "data_size",                  KSTAT_DATA_UINT64 },
 362         { "other_size",                 KSTAT_DATA_UINT64 },
 363         { "l2_hits",                    KSTAT_DATA_UINT64 },
 364         { "l2_misses",                  KSTAT_DATA_UINT64 },
 365         { "l2_feeds",                   KSTAT_DATA_UINT64 },
 366         { "l2_rw_clash",                KSTAT_DATA_UINT64 },
 367         { "l2_read_bytes",              KSTAT_DATA_UINT64 },
 368         { "l2_write_bytes",             KSTAT_DATA_UINT64 },
 369         { "l2_writes_sent",             KSTAT_DATA_UINT64 },
 370         { "l2_writes_done",             KSTAT_DATA_UINT64 },
 371         { "l2_writes_error",            KSTAT_DATA_UINT64 },
 372         { "l2_writes_hdr_miss",         KSTAT_DATA_UINT64 },
 373         { "l2_evict_lock_retry",        KSTAT_DATA_UINT64 },
 374         { "l2_evict_reading",           KSTAT_DATA_UINT64 },
 375         { "l2_free_on_write",           KSTAT_DATA_UINT64 },
 376         { "l2_abort_lowmem",            KSTAT_DATA_UINT64 },
 377         { "l2_cksum_bad",               KSTAT_DATA_UINT64 },
 378         { "l2_io_error",                KSTAT_DATA_UINT64 },
 379         { "l2_size",                    KSTAT_DATA_UINT64 },
 380         { "l2_asize",                   KSTAT_DATA_UINT64 },
 381         { "l2_hdr_size",                KSTAT_DATA_UINT64 },
 382         { "l2_compress_successes",      KSTAT_DATA_UINT64 },
 383         { "l2_compress_zeros",          KSTAT_DATA_UINT64 },
 384         { "l2_compress_failures",       KSTAT_DATA_UINT64 },
 385         { "memory_throttle_count",      KSTAT_DATA_UINT64 },
 386         { "duplicate_buffers",          KSTAT_DATA_UINT64 },
 387         { "duplicate_buffers_size",     KSTAT_DATA_UINT64 },
 388         { "duplicate_reads",            KSTAT_DATA_UINT64 },
 389         { "arc_meta_used",              KSTAT_DATA_UINT64 },
 390         { "arc_meta_limit",             KSTAT_DATA_UINT64 },
 391         { "arc_meta_max",               KSTAT_DATA_UINT64 }
 392 };
 393 
 394 #define ARCSTAT(stat)   (arc_stats.stat.value.ui64)
 395 
 396 #define ARCSTAT_INCR(stat, val) \
 397         atomic_add_64(&arc_stats.stat.value.ui64, (val))
 398 
 399 #define ARCSTAT_BUMP(stat)      ARCSTAT_INCR(stat, 1)
 400 #define ARCSTAT_BUMPDOWN(stat)  ARCSTAT_INCR(stat, -1)
 401 
 402 #define ARCSTAT_MAX(stat, val) {                                        \
 403         uint64_t m;                                                     \
 404         while ((val) > (m = arc_stats.stat.value.ui64) &&            \
 405             (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))     \
 406                 continue;                                               \
 407 }
 408 
 409 #define ARCSTAT_MAXSTAT(stat) \
 410         ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
 411 
 412 /*
 413  * We define a macro to allow ARC hits/misses to be easily broken down by
 414  * two separate conditions, giving a total of four different subtypes for
 415  * each of hits and misses (so eight statistics total).
 416  */
 417 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
 418         if (cond1) {                                                    \
 419                 if (cond2) {                                            \
 420                         ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
 421                 } else {                                                \
 422                         ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
 423                 }                                                       \
 424         } else {                                                        \
 425                 if (cond2) {                                            \
 426                         ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
 427                 } else {                                                \
 428                         ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
 429                 }                                                       \
 430         }
 431 
 432 kstat_t                 *arc_ksp;
 433 static arc_state_t      *arc_anon;
 434 static arc_state_t      *arc_mru;
 435 static arc_state_t      *arc_mru_ghost;
 436 static arc_state_t      *arc_mfu;
 437 static arc_state_t      *arc_mfu_ghost;
 438 static arc_state_t      *arc_l2c_only;
 439 
 440 /*
 441  * There are several ARC variables that are critical to export as kstats --
 442  * but we don't want to have to grovel around in the kstat whenever we wish to
 443  * manipulate them.  For these variables, we therefore define them to be in
 444  * terms of the statistic variable.  This assures that we are not introducing
 445  * the possibility of inconsistency by having shadow copies of the variables,
 446  * while still allowing the code to be readable.
 447  */
 448 #define arc_size        ARCSTAT(arcstat_size)   /* actual total arc size */
 449 #define arc_p           ARCSTAT(arcstat_p)      /* target size of MRU */
 450 #define arc_c           ARCSTAT(arcstat_c)      /* target size of cache */
 451 #define arc_c_min       ARCSTAT(arcstat_c_min)  /* min target cache size */
 452 #define arc_c_max       ARCSTAT(arcstat_c_max)  /* max target cache size */
 453 #define arc_meta_limit  ARCSTAT(arcstat_meta_limit) /* max size for metadata */
 454 #define arc_meta_used   ARCSTAT(arcstat_meta_used) /* size of metadata */
 455 #define arc_meta_max    ARCSTAT(arcstat_meta_max) /* max size of metadata */
 456 
 457 #define L2ARC_IS_VALID_COMPRESS(_c_) \
 458         ((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
 459 
 460 static int              arc_no_grow;    /* Don't try to grow cache size */
 461 static uint64_t         arc_tempreserve;
 462 static uint64_t         arc_loaned_bytes;
 463 
 464 typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
 465 
 466 typedef struct arc_callback arc_callback_t;
 467 
 468 struct arc_callback {
 469         void                    *acb_private;
 470         arc_done_func_t         *acb_done;
 471         arc_buf_t               *acb_buf;
 472         zio_t                   *acb_zio_dummy;
 473         arc_callback_t          *acb_next;
 474 };
 475 
 476 typedef struct arc_write_callback arc_write_callback_t;
 477 
 478 struct arc_write_callback {
 479         void            *awcb_private;
 480         arc_done_func_t *awcb_ready;
 481         arc_done_func_t *awcb_physdone;
 482         arc_done_func_t *awcb_done;
 483         arc_buf_t       *awcb_buf;
 484 };
 485 
 486 struct arc_buf_hdr {
 487         /* protected by hash lock */
 488         dva_t                   b_dva;
 489         uint64_t                b_birth;
 490         uint64_t                b_cksum0;
 491 
 492         kmutex_t                b_freeze_lock;
 493         zio_cksum_t             *b_freeze_cksum;
 494         void                    *b_thawed;
 495 
 496         arc_buf_hdr_t           *b_hash_next;
 497         arc_buf_t               *b_buf;
 498         uint32_t                b_flags;
 499         uint32_t                b_datacnt;
 500 
 501         arc_callback_t          *b_acb;
 502         kcondvar_t              b_cv;
 503 
 504         /* immutable */
 505         arc_buf_contents_t      b_type;
 506         uint64_t                b_size;
 507         uint64_t                b_spa;
 508 
 509         /* protected by arc state mutex */
 510         arc_state_t             *b_state;
 511         list_node_t             b_arc_node;
 512 
 513         /* updated atomically */
 514         clock_t                 b_arc_access;
 515 
 516         /* self protecting */
 517         refcount_t              b_refcnt;
 518 
 519         l2arc_buf_hdr_t         *b_l2hdr;
 520         list_node_t             b_l2node;
 521 };
 522 
 523 static arc_buf_t *arc_eviction_list;
 524 static kmutex_t arc_eviction_mtx;
 525 static arc_buf_hdr_t arc_eviction_hdr;
 526 static void arc_get_data_buf(arc_buf_t *buf);
 527 static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
 528 static int arc_evict_needed(arc_buf_contents_t type);
 529 static void arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes);
 530 static void arc_buf_watch(arc_buf_t *buf);
 531 
 532 static boolean_t l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab);
 533 
 534 #define GHOST_STATE(state)      \
 535         ((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||        \
 536         (state) == arc_l2c_only)
 537 
 538 /*
 539  * Private ARC flags.  These flags are private ARC only flags that will show up
 540  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
 541  * be passed in as arc_flags in things like arc_read.  However, these flags
 542  * should never be passed and should only be set by ARC code.  When adding new
 543  * public flags, make sure not to smash the private ones.
 544  */
 545 
 546 #define ARC_IN_HASH_TABLE       (1 << 9)  /* this buffer is hashed */
 547 #define ARC_IO_IN_PROGRESS      (1 << 10) /* I/O in progress for buf */
 548 #define ARC_IO_ERROR            (1 << 11) /* I/O failed for buf */
 549 #define ARC_FREED_IN_READ       (1 << 12) /* buf freed while in read */
 550 #define ARC_BUF_AVAILABLE       (1 << 13) /* block not in active use */
 551 #define ARC_INDIRECT            (1 << 14) /* this is an indirect block */
 552 #define ARC_FREE_IN_PROGRESS    (1 << 15) /* hdr about to be freed */
 553 #define ARC_L2_WRITING          (1 << 16) /* L2ARC write in progress */
 554 #define ARC_L2_EVICTED          (1 << 17) /* evicted during I/O */
 555 #define ARC_L2_WRITE_HEAD       (1 << 18) /* head of write list */
 556 
 557 #define HDR_IN_HASH_TABLE(hdr)  ((hdr)->b_flags & ARC_IN_HASH_TABLE)
 558 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS)
 559 #define HDR_IO_ERROR(hdr)       ((hdr)->b_flags & ARC_IO_ERROR)
 560 #define HDR_PREFETCH(hdr)       ((hdr)->b_flags & ARC_PREFETCH)
 561 #define HDR_FREED_IN_READ(hdr)  ((hdr)->b_flags & ARC_FREED_IN_READ)
 562 #define HDR_BUF_AVAILABLE(hdr)  ((hdr)->b_flags & ARC_BUF_AVAILABLE)
 563 #define HDR_FREE_IN_PROGRESS(hdr)       ((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
 564 #define HDR_L2CACHE(hdr)        ((hdr)->b_flags & ARC_L2CACHE)
 565 #define HDR_L2_READING(hdr)     ((hdr)->b_flags & ARC_IO_IN_PROGRESS &&  \
 566                                     (hdr)->b_l2hdr != NULL)
 567 #define HDR_L2_WRITING(hdr)     ((hdr)->b_flags & ARC_L2_WRITING)
 568 #define HDR_L2_EVICTED(hdr)     ((hdr)->b_flags & ARC_L2_EVICTED)
 569 #define HDR_L2_WRITE_HEAD(hdr)  ((hdr)->b_flags & ARC_L2_WRITE_HEAD)
 570 
 571 /*
 572  * Other sizes
 573  */
 574 
 575 #define HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
 576 #define L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
 577 
 578 /*
 579  * Hash table routines
 580  */
 581 
 582 #define HT_LOCK_PAD     64
 583 
 584 struct ht_lock {
 585         kmutex_t        ht_lock;
 586 #ifdef _KERNEL
 587         unsigned char   pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
 588 #endif
 589 };
 590 
 591 #define BUF_LOCKS 256
 592 typedef struct buf_hash_table {
 593         uint64_t ht_mask;
 594         arc_buf_hdr_t **ht_table;
 595         struct ht_lock ht_locks[BUF_LOCKS];
 596 } buf_hash_table_t;
 597 
 598 static buf_hash_table_t buf_hash_table;
 599 
 600 #define BUF_HASH_INDEX(spa, dva, birth) \
 601         (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
 602 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
 603 #define BUF_HASH_LOCK(idx)      (&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
 604 #define HDR_LOCK(hdr) \
 605         (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
 606 
 607 uint64_t zfs_crc64_table[256];
 608 
 609 /*
 610  * Level 2 ARC
 611  */
 612 
 613 #define L2ARC_WRITE_SIZE        (8 * 1024 * 1024)       /* initial write max */
 614 #define L2ARC_HEADROOM          2                       /* num of writes */
 615 /*
 616  * If we discover during ARC scan any buffers to be compressed, we boost
 617  * our headroom for the next scanning cycle by this percentage multiple.
 618  */
 619 #define L2ARC_HEADROOM_BOOST    200
 620 #define L2ARC_FEED_SECS         1               /* caching interval secs */
 621 #define L2ARC_FEED_MIN_MS       200             /* min caching interval ms */
 622 
 623 #define l2arc_writes_sent       ARCSTAT(arcstat_l2_writes_sent)
 624 #define l2arc_writes_done       ARCSTAT(arcstat_l2_writes_done)
 625 
 626 /* L2ARC Performance Tunables */
 627 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;    /* default max write size */
 628 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;  /* extra write during warmup */
 629 uint64_t l2arc_headroom = L2ARC_HEADROOM;       /* number of dev writes */
 630 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
 631 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;     /* interval seconds */
 632 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */
 633 boolean_t l2arc_noprefetch = B_TRUE;            /* don't cache prefetch bufs */
 634 boolean_t l2arc_feed_again = B_TRUE;            /* turbo warmup */
 635 boolean_t l2arc_norw = B_TRUE;                  /* no reads during writes */
 636 
 637 /*
 638  * L2ARC Internals
 639  */
 640 typedef struct l2arc_dev {
 641         vdev_t                  *l2ad_vdev;     /* vdev */
 642         spa_t                   *l2ad_spa;      /* spa */
 643         uint64_t                l2ad_hand;      /* next write location */
 644         uint64_t                l2ad_start;     /* first addr on device */
 645         uint64_t                l2ad_end;       /* last addr on device */
 646         uint64_t                l2ad_evict;     /* last addr eviction reached */
 647         boolean_t               l2ad_first;     /* first sweep through */
 648         boolean_t               l2ad_writing;   /* currently writing */
 649         list_t                  *l2ad_buflist;  /* buffer list */
 650         list_node_t             l2ad_node;      /* device list node */
 651 } l2arc_dev_t;
 652 
 653 static list_t L2ARC_dev_list;                   /* device list */
 654 static list_t *l2arc_dev_list;                  /* device list pointer */
 655 static kmutex_t l2arc_dev_mtx;                  /* device list mutex */
 656 static l2arc_dev_t *l2arc_dev_last;             /* last device used */
 657 static kmutex_t l2arc_buflist_mtx;              /* mutex for all buflists */
 658 static list_t L2ARC_free_on_write;              /* free after write buf list */
 659 static list_t *l2arc_free_on_write;             /* free after write list ptr */
 660 static kmutex_t l2arc_free_on_write_mtx;        /* mutex for list */
 661 static uint64_t l2arc_ndev;                     /* number of devices */
 662 
 663 typedef struct l2arc_read_callback {
 664         arc_buf_t               *l2rcb_buf;             /* read buffer */
 665         spa_t                   *l2rcb_spa;             /* spa */
 666         blkptr_t                l2rcb_bp;               /* original blkptr */
 667         zbookmark_t             l2rcb_zb;               /* original bookmark */
 668         int                     l2rcb_flags;            /* original flags */
 669         enum zio_compress       l2rcb_compress;         /* applied compress */
 670 } l2arc_read_callback_t;
 671 
 672 typedef struct l2arc_write_callback {
 673         l2arc_dev_t     *l2wcb_dev;             /* device info */
 674         arc_buf_hdr_t   *l2wcb_head;            /* head of write buflist */
 675 } l2arc_write_callback_t;
 676 
 677 struct l2arc_buf_hdr {
 678         /* protected by arc_buf_hdr  mutex */
 679         l2arc_dev_t             *b_dev;         /* L2ARC device */
 680         uint64_t                b_daddr;        /* disk address, offset byte */
 681         /* compression applied to buffer data */
 682         enum zio_compress       b_compress;
 683         /* real alloc'd buffer size depending on b_compress applied */
 684         int                     b_asize;
 685         /* temporary buffer holder for in-flight compressed data */
 686         void                    *b_tmp_cdata;
 687 };
 688 
 689 typedef struct l2arc_data_free {
 690         /* protected by l2arc_free_on_write_mtx */
 691         void            *l2df_data;
 692         size_t          l2df_size;
 693         void            (*l2df_func)(void *, size_t);
 694         list_node_t     l2df_list_node;
 695 } l2arc_data_free_t;
 696 
 697 static kmutex_t l2arc_feed_thr_lock;
 698 static kcondvar_t l2arc_feed_thr_cv;
 699 static uint8_t l2arc_thread_exit;
 700 
 701 static void l2arc_read_done(zio_t *zio);
 702 static void l2arc_hdr_stat_add(void);
 703 static void l2arc_hdr_stat_remove(void);
 704 
 705 static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr);
 706 static void l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr,
 707     enum zio_compress c);
 708 static void l2arc_release_cdata_buf(arc_buf_hdr_t *ab);
 709 
 710 static uint64_t
 711 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 712 {
 713         uint8_t *vdva = (uint8_t *)dva;
 714         uint64_t crc = -1ULL;
 715         int i;
 716 
 717         ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
 718 
 719         for (i = 0; i < sizeof (dva_t); i++)
 720                 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
 721 
 722         crc ^= (spa>>8) ^ birth;
 723 
 724         return (crc);
 725 }
 726 
 727 #define BUF_EMPTY(buf)                                          \
 728         ((buf)->b_dva.dva_word[0] == 0 &&                    \
 729         (buf)->b_dva.dva_word[1] == 0 &&                     \
 730         (buf)->b_birth == 0)
 731 
 732 #define BUF_EQUAL(spa, dva, birth, buf)                         \
 733         ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&       \
 734         ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&       \
 735         ((buf)->b_birth == birth) && ((buf)->b_spa == spa)
 736 
 737 static void
 738 buf_discard_identity(arc_buf_hdr_t *hdr)
 739 {
 740         hdr->b_dva.dva_word[0] = 0;
 741         hdr->b_dva.dva_word[1] = 0;
 742         hdr->b_birth = 0;
 743         hdr->b_cksum0 = 0;
 744 }
 745 
 746 static arc_buf_hdr_t *
 747 buf_hash_find(uint64_t spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
 748 {
 749         uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 750         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 751         arc_buf_hdr_t *buf;
 752 
 753         mutex_enter(hash_lock);
 754         for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
 755             buf = buf->b_hash_next) {
 756                 if (BUF_EQUAL(spa, dva, birth, buf)) {
 757                         *lockp = hash_lock;
 758                         return (buf);
 759                 }
 760         }
 761         mutex_exit(hash_lock);
 762         *lockp = NULL;
 763         return (NULL);
 764 }
 765 
 766 /*
 767  * Insert an entry into the hash table.  If there is already an element
 768  * equal to elem in the hash table, then the already existing element
 769  * will be returned and the new element will not be inserted.
 770  * Otherwise returns NULL.
 771  */
 772 static arc_buf_hdr_t *
 773 buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
 774 {
 775         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 776         kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 777         arc_buf_hdr_t *fbuf;
 778         uint32_t i;
 779 
 780         ASSERT(!HDR_IN_HASH_TABLE(buf));
 781         *lockp = hash_lock;
 782         mutex_enter(hash_lock);
 783         for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
 784             fbuf = fbuf->b_hash_next, i++) {
 785                 if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
 786                         return (fbuf);
 787         }
 788 
 789         buf->b_hash_next = buf_hash_table.ht_table[idx];
 790         buf_hash_table.ht_table[idx] = buf;
 791         buf->b_flags |= ARC_IN_HASH_TABLE;
 792 
 793         /* collect some hash table performance data */
 794         if (i > 0) {
 795                 ARCSTAT_BUMP(arcstat_hash_collisions);
 796                 if (i == 1)
 797                         ARCSTAT_BUMP(arcstat_hash_chains);
 798 
 799                 ARCSTAT_MAX(arcstat_hash_chain_max, i);
 800         }
 801 
 802         ARCSTAT_BUMP(arcstat_hash_elements);
 803         ARCSTAT_MAXSTAT(arcstat_hash_elements);
 804 
 805         return (NULL);
 806 }
 807 
 808 static void
 809 buf_hash_remove(arc_buf_hdr_t *buf)
 810 {
 811         arc_buf_hdr_t *fbuf, **bufp;
 812         uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
 813 
 814         ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
 815         ASSERT(HDR_IN_HASH_TABLE(buf));
 816 
 817         bufp = &buf_hash_table.ht_table[idx];
 818         while ((fbuf = *bufp) != buf) {
 819                 ASSERT(fbuf != NULL);
 820                 bufp = &fbuf->b_hash_next;
 821         }
 822         *bufp = buf->b_hash_next;
 823         buf->b_hash_next = NULL;
 824         buf->b_flags &= ~ARC_IN_HASH_TABLE;
 825 
 826         /* collect some hash table performance data */
 827         ARCSTAT_BUMPDOWN(arcstat_hash_elements);
 828 
 829         if (buf_hash_table.ht_table[idx] &&
 830             buf_hash_table.ht_table[idx]->b_hash_next == NULL)
 831                 ARCSTAT_BUMPDOWN(arcstat_hash_chains);
 832 }
 833 
 834 /*
 835  * Global data structures and functions for the buf kmem cache.
 836  */
 837 static kmem_cache_t *hdr_cache;
 838 static kmem_cache_t *buf_cache;
 839 
 840 static void
 841 buf_fini(void)
 842 {
 843         int i;
 844 
 845         kmem_free(buf_hash_table.ht_table,
 846             (buf_hash_table.ht_mask + 1) * sizeof (void *));
 847         for (i = 0; i < BUF_LOCKS; i++)
 848                 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
 849         kmem_cache_destroy(hdr_cache);
 850         kmem_cache_destroy(buf_cache);
 851 }
 852 
 853 /*
 854  * Constructor callback - called when the cache is empty
 855  * and a new buf is requested.
 856  */
 857 /* ARGSUSED */
 858 static int
 859 hdr_cons(void *vbuf, void *unused, int kmflag)
 860 {
 861         arc_buf_hdr_t *buf = vbuf;
 862 
 863         bzero(buf, sizeof (arc_buf_hdr_t));
 864         refcount_create(&buf->b_refcnt);
 865         cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
 866         mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
 867         arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 868 
 869         return (0);
 870 }
 871 
 872 /* ARGSUSED */
 873 static int
 874 buf_cons(void *vbuf, void *unused, int kmflag)
 875 {
 876         arc_buf_t *buf = vbuf;
 877 
 878         bzero(buf, sizeof (arc_buf_t));
 879         mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
 880         arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 881 
 882         return (0);
 883 }
 884 
 885 /*
 886  * Destructor callback - called when a cached buf is
 887  * no longer required.
 888  */
 889 /* ARGSUSED */
 890 static void
 891 hdr_dest(void *vbuf, void *unused)
 892 {
 893         arc_buf_hdr_t *buf = vbuf;
 894 
 895         ASSERT(BUF_EMPTY(buf));
 896         refcount_destroy(&buf->b_refcnt);
 897         cv_destroy(&buf->b_cv);
 898         mutex_destroy(&buf->b_freeze_lock);
 899         arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
 900 }
 901 
 902 /* ARGSUSED */
 903 static void
 904 buf_dest(void *vbuf, void *unused)
 905 {
 906         arc_buf_t *buf = vbuf;
 907 
 908         mutex_destroy(&buf->b_evict_lock);
 909         arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 910 }
 911 
 912 /*
 913  * Reclaim callback -- invoked when memory is low.
 914  */
 915 /* ARGSUSED */
 916 static void
 917 hdr_recl(void *unused)
 918 {
 919         dprintf("hdr_recl called\n");
 920         /*
 921          * umem calls the reclaim func when we destroy the buf cache,
 922          * which is after we do arc_fini().
 923          */
 924         if (!arc_dead)
 925                 cv_signal(&arc_reclaim_thr_cv);
 926 }
 927 
 928 static void
 929 buf_init(void)
 930 {
 931         uint64_t *ct;
 932         uint64_t hsize = 1ULL << 12;
 933         int i, j;
 934 
 935         /*
 936          * The hash table is big enough to fill all of physical memory
 937          * with an average 64K block size.  The table will take up
 938          * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
 939          */
 940         while (hsize * 65536 < physmem * PAGESIZE)
 941                 hsize <<= 1;
 942 retry:
 943         buf_hash_table.ht_mask = hsize - 1;
 944         buf_hash_table.ht_table =
 945             kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
 946         if (buf_hash_table.ht_table == NULL) {
 947                 ASSERT(hsize > (1ULL << 8));
 948                 hsize >>= 1;
 949                 goto retry;
 950         }
 951 
 952         hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
 953             0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
 954         buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 955             0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 956 
 957         for (i = 0; i < 256; i++)
 958                 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
 959                         *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
 960 
 961         for (i = 0; i < BUF_LOCKS; i++) {
 962                 mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
 963                     NULL, MUTEX_DEFAULT, NULL);
 964         }
 965 }
 966 
 967 #define ARC_MINTIME     (hz>>4) /* 62 ms */
 968 
 969 static void
 970 arc_cksum_verify(arc_buf_t *buf)
 971 {
 972         zio_cksum_t zc;
 973 
 974         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 975                 return;
 976 
 977         mutex_enter(&buf->b_hdr->b_freeze_lock);
 978         if (buf->b_hdr->b_freeze_cksum == NULL ||
 979             (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
 980                 mutex_exit(&buf->b_hdr->b_freeze_lock);
 981                 return;
 982         }
 983         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 984         if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 985                 panic("buffer modified while frozen!");
 986         mutex_exit(&buf->b_hdr->b_freeze_lock);
 987 }
 988 
 989 static int
 990 arc_cksum_equal(arc_buf_t *buf)
 991 {
 992         zio_cksum_t zc;
 993         int equal;
 994 
 995         mutex_enter(&buf->b_hdr->b_freeze_lock);
 996         fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 997         equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
 998         mutex_exit(&buf->b_hdr->b_freeze_lock);
 999 
1000         return (equal);
1001 }
1002 
1003 static void
1004 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1005 {
1006         if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1007                 return;
1008 
1009         mutex_enter(&buf->b_hdr->b_freeze_lock);
1010         if (buf->b_hdr->b_freeze_cksum != NULL) {
1011                 mutex_exit(&buf->b_hdr->b_freeze_lock);
1012                 return;
1013         }
1014         buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1015         fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1016             buf->b_hdr->b_freeze_cksum);
1017         mutex_exit(&buf->b_hdr->b_freeze_lock);
1018         arc_buf_watch(buf);
1019 }
1020 
1021 #ifndef _KERNEL
1022 typedef struct procctl {
1023         long cmd;
1024         prwatch_t prwatch;
1025 } procctl_t;
1026 #endif
1027 
1028 /* ARGSUSED */
1029 static void
1030 arc_buf_unwatch(arc_buf_t *buf)
1031 {
1032 #ifndef _KERNEL
1033         if (arc_watch) {
1034                 int result;
1035                 procctl_t ctl;
1036                 ctl.cmd = PCWATCH;
1037                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1038                 ctl.prwatch.pr_size = 0;
1039                 ctl.prwatch.pr_wflags = 0;
1040                 result = write(arc_procfd, &ctl, sizeof (ctl));
1041                 ASSERT3U(result, ==, sizeof (ctl));
1042         }
1043 #endif
1044 }
1045 
1046 /* ARGSUSED */
1047 static void
1048 arc_buf_watch(arc_buf_t *buf)
1049 {
1050 #ifndef _KERNEL
1051         if (arc_watch) {
1052                 int result;
1053                 procctl_t ctl;
1054                 ctl.cmd = PCWATCH;
1055                 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1056                 ctl.prwatch.pr_size = buf->b_hdr->b_size;
1057                 ctl.prwatch.pr_wflags = WA_WRITE;
1058                 result = write(arc_procfd, &ctl, sizeof (ctl));
1059                 ASSERT3U(result, ==, sizeof (ctl));
1060         }
1061 #endif
1062 }
1063 
1064 void
1065 arc_buf_thaw(arc_buf_t *buf)
1066 {
1067         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1068                 if (buf->b_hdr->b_state != arc_anon)
1069                         panic("modifying non-anon buffer!");
1070                 if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
1071                         panic("modifying buffer while i/o in progress!");
1072                 arc_cksum_verify(buf);
1073         }
1074 
1075         mutex_enter(&buf->b_hdr->b_freeze_lock);
1076         if (buf->b_hdr->b_freeze_cksum != NULL) {
1077                 kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1078                 buf->b_hdr->b_freeze_cksum = NULL;
1079         }
1080 
1081         if (zfs_flags & ZFS_DEBUG_MODIFY) {
1082                 if (buf->b_hdr->b_thawed)
1083                         kmem_free(buf->b_hdr->b_thawed, 1);
1084                 buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
1085         }
1086 
1087         mutex_exit(&buf->b_hdr->b_freeze_lock);
1088 
1089         arc_buf_unwatch(buf);
1090 }
1091 
1092 void
1093 arc_buf_freeze(arc_buf_t *buf)
1094 {
1095         kmutex_t *hash_lock;
1096 
1097         if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1098                 return;
1099 
1100         hash_lock = HDR_LOCK(buf->b_hdr);
1101         mutex_enter(hash_lock);
1102 
1103         ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1104             buf->b_hdr->b_state == arc_anon);
1105         arc_cksum_compute(buf, B_FALSE);
1106         mutex_exit(hash_lock);
1107 
1108 }
1109 
1110 static void
1111 add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1112 {
1113         ASSERT(MUTEX_HELD(hash_lock));
1114 
1115         if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
1116             (ab->b_state != arc_anon)) {
1117                 uint64_t delta = ab->b_size * ab->b_datacnt;
1118                 list_t *list = &ab->b_state->arcs_list[ab->b_type];
1119                 uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
1120 
1121                 ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
1122                 mutex_enter(&ab->b_state->arcs_mtx);
1123                 ASSERT(list_link_active(&ab->b_arc_node));
1124                 list_remove(list, ab);
1125                 if (GHOST_STATE(ab->b_state)) {
1126                         ASSERT0(ab->b_datacnt);
1127                         ASSERT3P(ab->b_buf, ==, NULL);
1128                         delta = ab->b_size;
1129                 }
1130                 ASSERT(delta > 0);
1131                 ASSERT3U(*size, >=, delta);
1132                 atomic_add_64(size, -delta);
1133                 mutex_exit(&ab->b_state->arcs_mtx);
1134                 /* remove the prefetch flag if we get a reference */
1135                 if (ab->b_flags & ARC_PREFETCH)
1136                         ab->b_flags &= ~ARC_PREFETCH;
1137         }
1138 }
1139 
1140 static int
1141 remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
1142 {
1143         int cnt;
1144         arc_state_t *state = ab->b_state;
1145 
1146         ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1147         ASSERT(!GHOST_STATE(state));
1148 
1149         if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
1150             (state != arc_anon)) {
1151                 uint64_t *size = &state->arcs_lsize[ab->b_type];
1152 
1153                 ASSERT(!MUTEX_HELD(&state->arcs_mtx));
1154                 mutex_enter(&state->arcs_mtx);
1155                 ASSERT(!list_link_active(&ab->b_arc_node));
1156                 list_insert_head(&state->arcs_list[ab->b_type], ab);
1157                 ASSERT(ab->b_datacnt > 0);
1158                 atomic_add_64(size, ab->b_size * ab->b_datacnt);
1159                 mutex_exit(&state->arcs_mtx);
1160         }
1161         return (cnt);
1162 }
1163 
1164 /*
1165  * Move the supplied buffer to the indicated state.  The mutex
1166  * for the buffer must be held by the caller.
1167  */
1168 static void
1169 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
1170 {
1171         arc_state_t *old_state = ab->b_state;
1172         int64_t refcnt = refcount_count(&ab->b_refcnt);
1173         uint64_t from_delta, to_delta;
1174 
1175         ASSERT(MUTEX_HELD(hash_lock));
1176         ASSERT3P(new_state, !=, old_state);
1177         ASSERT(refcnt == 0 || ab->b_datacnt > 0);
1178         ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
1179         ASSERT(ab->b_datacnt <= 1 || old_state != arc_anon);
1180 
1181         from_delta = to_delta = ab->b_datacnt * ab->b_size;
1182 
1183         /*
1184          * If this buffer is evictable, transfer it from the
1185          * old state list to the new state list.
1186          */
1187         if (refcnt == 0) {
1188                 if (old_state != arc_anon) {
1189                         int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
1190                         uint64_t *size = &old_state->arcs_lsize[ab->b_type];
1191 
1192                         if (use_mutex)
1193                                 mutex_enter(&old_state->arcs_mtx);
1194 
1195                         ASSERT(list_link_active(&ab->b_arc_node));
1196                         list_remove(&old_state->arcs_list[ab->b_type], ab);
1197 
1198                         /*
1199                          * If prefetching out of the ghost cache,
1200                          * we will have a non-zero datacnt.
1201                          */
1202                         if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
1203                                 /* ghost elements have a ghost size */
1204                                 ASSERT(ab->b_buf == NULL);
1205                                 from_delta = ab->b_size;
1206                         }
1207                         ASSERT3U(*size, >=, from_delta);
1208                         atomic_add_64(size, -from_delta);
1209 
1210                         if (use_mutex)
1211                                 mutex_exit(&old_state->arcs_mtx);
1212                 }
1213                 if (new_state != arc_anon) {
1214                         int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
1215                         uint64_t *size = &new_state->arcs_lsize[ab->b_type];
1216 
1217                         if (use_mutex)
1218                                 mutex_enter(&new_state->arcs_mtx);
1219 
1220                         list_insert_head(&new_state->arcs_list[ab->b_type], ab);
1221 
1222                         /* ghost elements have a ghost size */
1223                         if (GHOST_STATE(new_state)) {
1224                                 ASSERT(ab->b_datacnt == 0);
1225                                 ASSERT(ab->b_buf == NULL);
1226                                 to_delta = ab->b_size;
1227                         }
1228                         atomic_add_64(size, to_delta);
1229 
1230                         if (use_mutex)
1231                                 mutex_exit(&new_state->arcs_mtx);
1232                 }
1233         }
1234 
1235         ASSERT(!BUF_EMPTY(ab));
1236         if (new_state == arc_anon && HDR_IN_HASH_TABLE(ab))
1237                 buf_hash_remove(ab);
1238 
1239         /* adjust state sizes */
1240         if (to_delta)
1241                 atomic_add_64(&new_state->arcs_size, to_delta);
1242         if (from_delta) {
1243                 ASSERT3U(old_state->arcs_size, >=, from_delta);
1244                 atomic_add_64(&old_state->arcs_size, -from_delta);
1245         }
1246         ab->b_state = new_state;
1247 
1248         /* adjust l2arc hdr stats */
1249         if (new_state == arc_l2c_only)
1250                 l2arc_hdr_stat_add();
1251         else if (old_state == arc_l2c_only)
1252                 l2arc_hdr_stat_remove();
1253 }
1254 
1255 void
1256 arc_space_consume(uint64_t space, arc_space_type_t type)
1257 {
1258         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1259 
1260         switch (type) {
1261         case ARC_SPACE_DATA:
1262                 ARCSTAT_INCR(arcstat_data_size, space);
1263                 break;
1264         case ARC_SPACE_OTHER:
1265                 ARCSTAT_INCR(arcstat_other_size, space);
1266                 break;
1267         case ARC_SPACE_HDRS:
1268                 ARCSTAT_INCR(arcstat_hdr_size, space);
1269                 break;
1270         case ARC_SPACE_L2HDRS:
1271                 ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1272                 break;
1273         }
1274 
1275         ARCSTAT_INCR(arcstat_meta_used, space);
1276         atomic_add_64(&arc_size, space);
1277 }
1278 
1279 void
1280 arc_space_return(uint64_t space, arc_space_type_t type)
1281 {
1282         ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1283 
1284         switch (type) {
1285         case ARC_SPACE_DATA:
1286                 ARCSTAT_INCR(arcstat_data_size, -space);
1287                 break;
1288         case ARC_SPACE_OTHER:
1289                 ARCSTAT_INCR(arcstat_other_size, -space);
1290                 break;
1291         case ARC_SPACE_HDRS:
1292                 ARCSTAT_INCR(arcstat_hdr_size, -space);
1293                 break;
1294         case ARC_SPACE_L2HDRS:
1295                 ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1296                 break;
1297         }
1298 
1299         ASSERT(arc_meta_used >= space);
1300         if (arc_meta_max < arc_meta_used)
1301                 arc_meta_max = arc_meta_used;
1302         ARCSTAT_INCR(arcstat_meta_used, -space);
1303         ASSERT(arc_size >= space);
1304         atomic_add_64(&arc_size, -space);
1305 }
1306 
1307 void *
1308 arc_data_buf_alloc(uint64_t size)
1309 {
1310         if (arc_evict_needed(ARC_BUFC_DATA))
1311                 cv_signal(&arc_reclaim_thr_cv);
1312         atomic_add_64(&arc_size, size);
1313         return (zio_data_buf_alloc(size));
1314 }
1315 
1316 void
1317 arc_data_buf_free(void *buf, uint64_t size)
1318 {
1319         zio_data_buf_free(buf, size);
1320         ASSERT(arc_size >= size);
1321         atomic_add_64(&arc_size, -size);
1322 }
1323 
1324 arc_buf_t *
1325 arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
1326 {
1327         arc_buf_hdr_t *hdr;
1328         arc_buf_t *buf;
1329 
1330         ASSERT3U(size, >, 0);
1331         hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
1332         ASSERT(BUF_EMPTY(hdr));
1333         hdr->b_size = size;
1334         hdr->b_type = type;
1335         hdr->b_spa = spa_load_guid(spa);
1336         hdr->b_state = arc_anon;
1337         hdr->b_arc_access = 0;
1338         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1339         buf->b_hdr = hdr;
1340         buf->b_data = NULL;
1341         buf->b_efunc = NULL;
1342         buf->b_private = NULL;
1343         buf->b_next = NULL;
1344         hdr->b_buf = buf;
1345         arc_get_data_buf(buf);
1346         hdr->b_datacnt = 1;
1347         hdr->b_flags = 0;
1348         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1349         (void) refcount_add(&hdr->b_refcnt, tag);
1350 
1351         return (buf);
1352 }
1353 
1354 static char *arc_onloan_tag = "onloan";
1355 
1356 /*
1357  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1358  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1359  * buffers must be returned to the arc before they can be used by the DMU or
1360  * freed.
1361  */
1362 arc_buf_t *
1363 arc_loan_buf(spa_t *spa, int size)
1364 {
1365         arc_buf_t *buf;
1366 
1367         buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1368 
1369         atomic_add_64(&arc_loaned_bytes, size);
1370         return (buf);
1371 }
1372 
1373 /*
1374  * Return a loaned arc buffer to the arc.
1375  */
1376 void
1377 arc_return_buf(arc_buf_t *buf, void *tag)
1378 {
1379         arc_buf_hdr_t *hdr = buf->b_hdr;
1380 
1381         ASSERT(buf->b_data != NULL);
1382         (void) refcount_add(&hdr->b_refcnt, tag);
1383         (void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
1384 
1385         atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1386 }
1387 
1388 /* Detach an arc_buf from a dbuf (tag) */
1389 void
1390 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1391 {
1392         arc_buf_hdr_t *hdr;
1393 
1394         ASSERT(buf->b_data != NULL);
1395         hdr = buf->b_hdr;
1396         (void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
1397         (void) refcount_remove(&hdr->b_refcnt, tag);
1398         buf->b_efunc = NULL;
1399         buf->b_private = NULL;
1400 
1401         atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1402 }
1403 
1404 static arc_buf_t *
1405 arc_buf_clone(arc_buf_t *from)
1406 {
1407         arc_buf_t *buf;
1408         arc_buf_hdr_t *hdr = from->b_hdr;
1409         uint64_t size = hdr->b_size;
1410 
1411         ASSERT(hdr->b_state != arc_anon);
1412 
1413         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1414         buf->b_hdr = hdr;
1415         buf->b_data = NULL;
1416         buf->b_efunc = NULL;
1417         buf->b_private = NULL;
1418         buf->b_next = hdr->b_buf;
1419         hdr->b_buf = buf;
1420         arc_get_data_buf(buf);
1421         bcopy(from->b_data, buf->b_data, size);
1422 
1423         /*
1424          * This buffer already exists in the arc so create a duplicate
1425          * copy for the caller.  If the buffer is associated with user data
1426          * then track the size and number of duplicates.  These stats will be
1427          * updated as duplicate buffers are created and destroyed.
1428          */
1429         if (hdr->b_type == ARC_BUFC_DATA) {
1430                 ARCSTAT_BUMP(arcstat_duplicate_buffers);
1431                 ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1432         }
1433         hdr->b_datacnt += 1;
1434         return (buf);
1435 }
1436 
1437 void
1438 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1439 {
1440         arc_buf_hdr_t *hdr;
1441         kmutex_t *hash_lock;
1442 
1443         /*
1444          * Check to see if this buffer is evicted.  Callers
1445          * must verify b_data != NULL to know if the add_ref
1446          * was successful.
1447          */
1448         mutex_enter(&buf->b_evict_lock);
1449         if (buf->b_data == NULL) {
1450                 mutex_exit(&buf->b_evict_lock);
1451                 return;
1452         }
1453         hash_lock = HDR_LOCK(buf->b_hdr);
1454         mutex_enter(hash_lock);
1455         hdr = buf->b_hdr;
1456         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1457         mutex_exit(&buf->b_evict_lock);
1458 
1459         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
1460         add_reference(hdr, hash_lock, tag);
1461         DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1462         arc_access(hdr, hash_lock);
1463         mutex_exit(hash_lock);
1464         ARCSTAT_BUMP(arcstat_hits);
1465         ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
1466             demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
1467             data, metadata, hits);
1468 }
1469 
1470 /*
1471  * Free the arc data buffer.  If it is an l2arc write in progress,
1472  * the buffer is placed on l2arc_free_on_write to be freed later.
1473  */
1474 static void
1475 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1476 {
1477         arc_buf_hdr_t *hdr = buf->b_hdr;
1478 
1479         if (HDR_L2_WRITING(hdr)) {
1480                 l2arc_data_free_t *df;
1481                 df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
1482                 df->l2df_data = buf->b_data;
1483                 df->l2df_size = hdr->b_size;
1484                 df->l2df_func = free_func;
1485                 mutex_enter(&l2arc_free_on_write_mtx);
1486                 list_insert_head(l2arc_free_on_write, df);
1487                 mutex_exit(&l2arc_free_on_write_mtx);
1488                 ARCSTAT_BUMP(arcstat_l2_free_on_write);
1489         } else {
1490                 free_func(buf->b_data, hdr->b_size);
1491         }
1492 }
1493 
1494 static void
1495 arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
1496 {
1497         arc_buf_t **bufp;
1498 
1499         /* free up data associated with the buf */
1500         if (buf->b_data) {
1501                 arc_state_t *state = buf->b_hdr->b_state;
1502                 uint64_t size = buf->b_hdr->b_size;
1503                 arc_buf_contents_t type = buf->b_hdr->b_type;
1504 
1505                 arc_cksum_verify(buf);
1506                 arc_buf_unwatch(buf);
1507 
1508                 if (!recycle) {
1509                         if (type == ARC_BUFC_METADATA) {
1510                                 arc_buf_data_free(buf, zio_buf_free);
1511                                 arc_space_return(size, ARC_SPACE_DATA);
1512                         } else {
1513                                 ASSERT(type == ARC_BUFC_DATA);
1514                                 arc_buf_data_free(buf, zio_data_buf_free);
1515                                 ARCSTAT_INCR(arcstat_data_size, -size);
1516                                 atomic_add_64(&arc_size, -size);
1517                         }
1518                 }
1519                 if (list_link_active(&buf->b_hdr->b_arc_node)) {
1520                         uint64_t *cnt = &state->arcs_lsize[type];
1521 
1522                         ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
1523                         ASSERT(state != arc_anon);
1524 
1525                         ASSERT3U(*cnt, >=, size);
1526                         atomic_add_64(cnt, -size);
1527                 }
1528                 ASSERT3U(state->arcs_size, >=, size);
1529                 atomic_add_64(&state->arcs_size, -size);
1530                 buf->b_data = NULL;
1531 
1532                 /*
1533                  * If we're destroying a duplicate buffer make sure
1534                  * that the appropriate statistics are updated.
1535                  */
1536                 if (buf->b_hdr->b_datacnt > 1 &&
1537                     buf->b_hdr->b_type == ARC_BUFC_DATA) {
1538                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
1539                         ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
1540                 }
1541                 ASSERT(buf->b_hdr->b_datacnt > 0);
1542                 buf->b_hdr->b_datacnt -= 1;
1543         }
1544 
1545         /* only remove the buf if requested */
1546         if (!all)
1547                 return;
1548 
1549         /* remove the buf from the hdr list */
1550         for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
1551                 continue;
1552         *bufp = buf->b_next;
1553         buf->b_next = NULL;
1554 
1555         ASSERT(buf->b_efunc == NULL);
1556 
1557         /* clean up the buf */
1558         buf->b_hdr = NULL;
1559         kmem_cache_free(buf_cache, buf);
1560 }
1561 
1562 static void
1563 arc_hdr_destroy(arc_buf_hdr_t *hdr)
1564 {
1565         ASSERT(refcount_is_zero(&hdr->b_refcnt));
1566         ASSERT3P(hdr->b_state, ==, arc_anon);
1567         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1568         l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
1569 
1570         if (l2hdr != NULL) {
1571                 boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
1572                 /*
1573                  * To prevent arc_free() and l2arc_evict() from
1574                  * attempting to free the same buffer at the same time,
1575                  * a FREE_IN_PROGRESS flag is given to arc_free() to
1576                  * give it priority.  l2arc_evict() can't destroy this
1577                  * header while we are waiting on l2arc_buflist_mtx.
1578                  *
1579                  * The hdr may be removed from l2ad_buflist before we
1580                  * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
1581                  */
1582                 if (!buflist_held) {
1583                         mutex_enter(&l2arc_buflist_mtx);
1584                         l2hdr = hdr->b_l2hdr;
1585                 }
1586 
1587                 if (l2hdr != NULL) {
1588                         list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
1589                         ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
1590                         ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
1591                         kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
1592                         if (hdr->b_state == arc_l2c_only)
1593                                 l2arc_hdr_stat_remove();
1594                         hdr->b_l2hdr = NULL;
1595                 }
1596 
1597                 if (!buflist_held)
1598                         mutex_exit(&l2arc_buflist_mtx);
1599         }
1600 
1601         if (!BUF_EMPTY(hdr)) {
1602                 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1603                 buf_discard_identity(hdr);
1604         }
1605         while (hdr->b_buf) {
1606                 arc_buf_t *buf = hdr->b_buf;
1607 
1608                 if (buf->b_efunc) {
1609                         mutex_enter(&arc_eviction_mtx);
1610                         mutex_enter(&buf->b_evict_lock);
1611                         ASSERT(buf->b_hdr != NULL);
1612                         arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
1613                         hdr->b_buf = buf->b_next;
1614                         buf->b_hdr = &arc_eviction_hdr;
1615                         buf->b_next = arc_eviction_list;
1616                         arc_eviction_list = buf;
1617                         mutex_exit(&buf->b_evict_lock);
1618                         mutex_exit(&arc_eviction_mtx);
1619                 } else {
1620                         arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
1621                 }
1622         }
1623         if (hdr->b_freeze_cksum != NULL) {
1624                 kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1625                 hdr->b_freeze_cksum = NULL;
1626         }
1627         if (hdr->b_thawed) {
1628                 kmem_free(hdr->b_thawed, 1);
1629                 hdr->b_thawed = NULL;
1630         }
1631 
1632         ASSERT(!list_link_active(&hdr->b_arc_node));
1633         ASSERT3P(hdr->b_hash_next, ==, NULL);
1634         ASSERT3P(hdr->b_acb, ==, NULL);
1635         kmem_cache_free(hdr_cache, hdr);
1636 }
1637 
1638 void
1639 arc_buf_free(arc_buf_t *buf, void *tag)
1640 {
1641         arc_buf_hdr_t *hdr = buf->b_hdr;
1642         int hashed = hdr->b_state != arc_anon;
1643 
1644         ASSERT(buf->b_efunc == NULL);
1645         ASSERT(buf->b_data != NULL);
1646 
1647         if (hashed) {
1648                 kmutex_t *hash_lock = HDR_LOCK(hdr);
1649 
1650                 mutex_enter(hash_lock);
1651                 hdr = buf->b_hdr;
1652                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1653 
1654                 (void) remove_reference(hdr, hash_lock, tag);
1655                 if (hdr->b_datacnt > 1) {
1656                         arc_buf_destroy(buf, FALSE, TRUE);
1657                 } else {
1658                         ASSERT(buf == hdr->b_buf);
1659                         ASSERT(buf->b_efunc == NULL);
1660                         hdr->b_flags |= ARC_BUF_AVAILABLE;
1661                 }
1662                 mutex_exit(hash_lock);
1663         } else if (HDR_IO_IN_PROGRESS(hdr)) {
1664                 int destroy_hdr;
1665                 /*
1666                  * We are in the middle of an async write.  Don't destroy
1667                  * this buffer unless the write completes before we finish
1668                  * decrementing the reference count.
1669                  */
1670                 mutex_enter(&arc_eviction_mtx);
1671                 (void) remove_reference(hdr, NULL, tag);
1672                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
1673                 destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
1674                 mutex_exit(&arc_eviction_mtx);
1675                 if (destroy_hdr)
1676                         arc_hdr_destroy(hdr);
1677         } else {
1678                 if (remove_reference(hdr, NULL, tag) > 0)
1679                         arc_buf_destroy(buf, FALSE, TRUE);
1680                 else
1681                         arc_hdr_destroy(hdr);
1682         }
1683 }
1684 
1685 boolean_t
1686 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
1687 {
1688         arc_buf_hdr_t *hdr = buf->b_hdr;
1689         kmutex_t *hash_lock = HDR_LOCK(hdr);
1690         boolean_t no_callback = (buf->b_efunc == NULL);
1691 
1692         if (hdr->b_state == arc_anon) {
1693                 ASSERT(hdr->b_datacnt == 1);
1694                 arc_buf_free(buf, tag);
1695                 return (no_callback);
1696         }
1697 
1698         mutex_enter(hash_lock);
1699         hdr = buf->b_hdr;
1700         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1701         ASSERT(hdr->b_state != arc_anon);
1702         ASSERT(buf->b_data != NULL);
1703 
1704         (void) remove_reference(hdr, hash_lock, tag);
1705         if (hdr->b_datacnt > 1) {
1706                 if (no_callback)
1707                         arc_buf_destroy(buf, FALSE, TRUE);
1708         } else if (no_callback) {
1709                 ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
1710                 ASSERT(buf->b_efunc == NULL);
1711                 hdr->b_flags |= ARC_BUF_AVAILABLE;
1712         }
1713         ASSERT(no_callback || hdr->b_datacnt > 1 ||
1714             refcount_is_zero(&hdr->b_refcnt));
1715         mutex_exit(hash_lock);
1716         return (no_callback);
1717 }
1718 
1719 int
1720 arc_buf_size(arc_buf_t *buf)
1721 {
1722         return (buf->b_hdr->b_size);
1723 }
1724 
1725 /*
1726  * Called from the DMU to determine if the current buffer should be
1727  * evicted. In order to ensure proper locking, the eviction must be initiated
1728  * from the DMU. Return true if the buffer is associated with user data and
1729  * duplicate buffers still exist.
1730  */
1731 boolean_t
1732 arc_buf_eviction_needed(arc_buf_t *buf)
1733 {
1734         arc_buf_hdr_t *hdr;
1735         boolean_t evict_needed = B_FALSE;
1736 
1737         if (zfs_disable_dup_eviction)
1738                 return (B_FALSE);
1739 
1740         mutex_enter(&buf->b_evict_lock);
1741         hdr = buf->b_hdr;
1742         if (hdr == NULL) {
1743                 /*
1744                  * We are in arc_do_user_evicts(); let that function
1745                  * perform the eviction.
1746                  */
1747                 ASSERT(buf->b_data == NULL);
1748                 mutex_exit(&buf->b_evict_lock);
1749                 return (B_FALSE);
1750         } else if (buf->b_data == NULL) {
1751                 /*
1752                  * We have already been added to the arc eviction list;
1753                  * recommend eviction.
1754                  */
1755                 ASSERT3P(hdr, ==, &arc_eviction_hdr);
1756                 mutex_exit(&buf->b_evict_lock);
1757                 return (B_TRUE);
1758         }
1759 
1760         if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
1761                 evict_needed = B_TRUE;
1762 
1763         mutex_exit(&buf->b_evict_lock);
1764         return (evict_needed);
1765 }
1766 
1767 /*
1768  * Evict buffers from list until we've removed the specified number of
1769  * bytes.  Move the removed buffers to the appropriate evict state.
1770  * If the recycle flag is set, then attempt to "recycle" a buffer:
1771  * - look for a buffer to evict that is `bytes' long.
1772  * - return the data block from this buffer rather than freeing it.
1773  * This flag is used by callers that are trying to make space for a
1774  * new buffer in a full arc cache.
1775  *
1776  * This function makes a "best effort".  It skips over any buffers
1777  * it can't get a hash_lock on, and so may not catch all candidates.
1778  * It may also return without evicting as much space as requested.
1779  */
1780 static void *
1781 arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
1782     arc_buf_contents_t type)
1783 {
1784         arc_state_t *evicted_state;
1785         uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
1786         arc_buf_hdr_t *ab, *ab_prev = NULL;
1787         list_t *list = &state->arcs_list[type];
1788         kmutex_t *hash_lock;
1789         boolean_t have_lock;
1790         void *stolen = NULL;
1791         arc_buf_hdr_t marker = { 0 };
1792         int count = 0;
1793 
1794         ASSERT(state == arc_mru || state == arc_mfu);
1795 
1796         evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1797 
1798         mutex_enter(&state->arcs_mtx);
1799         mutex_enter(&evicted_state->arcs_mtx);
1800 
1801         for (ab = list_tail(list); ab; ab = ab_prev) {
1802                 ab_prev = list_prev(list, ab);
1803                 /* prefetch buffers have a minimum lifespan */
1804                 if (HDR_IO_IN_PROGRESS(ab) ||
1805                     (spa && ab->b_spa != spa) ||
1806                     (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
1807                     ddi_get_lbolt() - ab->b_arc_access <
1808                     arc_min_prefetch_lifespan)) {
1809                         skipped++;
1810                         continue;
1811                 }
1812                 /* "lookahead" for better eviction candidate */
1813                 if (recycle && ab->b_size != bytes &&
1814                     ab_prev && ab_prev->b_size == bytes)
1815                         continue;
1816 
1817                 /* ignore markers */
1818                 if (ab->b_spa == 0)
1819                         continue;
1820 
1821                 /*
1822                  * It may take a long time to evict all the bufs requested.
1823                  * To avoid blocking all arc activity, periodically drop
1824                  * the arcs_mtx and give other threads a chance to run
1825                  * before reacquiring the lock.
1826                  *
1827                  * If we are looking for a buffer to recycle, we are in
1828                  * the hot code path, so don't sleep.
1829                  */
1830                 if (!recycle && count++ > arc_evict_iterations) {
1831                         list_insert_after(list, ab, &marker);
1832                         mutex_exit(&evicted_state->arcs_mtx);
1833                         mutex_exit(&state->arcs_mtx);
1834                         kpreempt(KPREEMPT_SYNC);
1835                         mutex_enter(&state->arcs_mtx);
1836                         mutex_enter(&evicted_state->arcs_mtx);
1837                         ab_prev = list_prev(list, &marker);
1838                         list_remove(list, &marker);
1839                         count = 0;
1840                         continue;
1841                 }
1842 
1843                 hash_lock = HDR_LOCK(ab);
1844                 have_lock = MUTEX_HELD(hash_lock);
1845                 if (have_lock || mutex_tryenter(hash_lock)) {
1846                         ASSERT0(refcount_count(&ab->b_refcnt));
1847                         ASSERT(ab->b_datacnt > 0);
1848                         while (ab->b_buf) {
1849                                 arc_buf_t *buf = ab->b_buf;
1850                                 if (!mutex_tryenter(&buf->b_evict_lock)) {
1851                                         missed += 1;
1852                                         break;
1853                                 }
1854                                 if (buf->b_data) {
1855                                         bytes_evicted += ab->b_size;
1856                                         if (recycle && ab->b_type == type &&
1857                                             ab->b_size == bytes &&
1858                                             !HDR_L2_WRITING(ab)) {
1859                                                 stolen = buf->b_data;
1860                                                 recycle = FALSE;
1861                                         }
1862                                 }
1863                                 if (buf->b_efunc) {
1864                                         mutex_enter(&arc_eviction_mtx);
1865                                         arc_buf_destroy(buf,
1866                                             buf->b_data == stolen, FALSE);
1867                                         ab->b_buf = buf->b_next;
1868                                         buf->b_hdr = &arc_eviction_hdr;
1869                                         buf->b_next = arc_eviction_list;
1870                                         arc_eviction_list = buf;
1871                                         mutex_exit(&arc_eviction_mtx);
1872                                         mutex_exit(&buf->b_evict_lock);
1873                                 } else {
1874                                         mutex_exit(&buf->b_evict_lock);
1875                                         arc_buf_destroy(buf,
1876                                             buf->b_data == stolen, TRUE);
1877                                 }
1878                         }
1879 
1880                         if (ab->b_l2hdr) {
1881                                 ARCSTAT_INCR(arcstat_evict_l2_cached,
1882                                     ab->b_size);
1883                         } else {
1884                                 if (l2arc_write_eligible(ab->b_spa, ab)) {
1885                                         ARCSTAT_INCR(arcstat_evict_l2_eligible,
1886                                             ab->b_size);
1887                                 } else {
1888                                         ARCSTAT_INCR(
1889                                             arcstat_evict_l2_ineligible,
1890                                             ab->b_size);
1891                                 }
1892                         }
1893 
1894                         if (ab->b_datacnt == 0) {
1895                                 arc_change_state(evicted_state, ab, hash_lock);
1896                                 ASSERT(HDR_IN_HASH_TABLE(ab));
1897                                 ab->b_flags |= ARC_IN_HASH_TABLE;
1898                                 ab->b_flags &= ~ARC_BUF_AVAILABLE;
1899                                 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
1900                         }
1901                         if (!have_lock)
1902                                 mutex_exit(hash_lock);
1903                         if (bytes >= 0 && bytes_evicted >= bytes)
1904                                 break;
1905                 } else {
1906                         missed += 1;
1907                 }
1908         }
1909 
1910         mutex_exit(&evicted_state->arcs_mtx);
1911         mutex_exit(&state->arcs_mtx);
1912 
1913         if (bytes_evicted < bytes)
1914                 dprintf("only evicted %lld bytes from %x",
1915                     (longlong_t)bytes_evicted, state);
1916 
1917         if (skipped)
1918                 ARCSTAT_INCR(arcstat_evict_skip, skipped);
1919 
1920         if (missed)
1921                 ARCSTAT_INCR(arcstat_mutex_miss, missed);
1922 
1923         /*
1924          * Note: we have just evicted some data into the ghost state,
1925          * potentially putting the ghost size over the desired size.  Rather
1926          * that evicting from the ghost list in this hot code path, leave
1927          * this chore to the arc_reclaim_thread().
1928          */
1929 
1930         return (stolen);
1931 }
1932 
1933 /*
1934  * Remove buffers from list until we've removed the specified number of
1935  * bytes.  Destroy the buffers that are removed.
1936  */
1937 static void
1938 arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
1939 {
1940         arc_buf_hdr_t *ab, *ab_prev;
1941         arc_buf_hdr_t marker = { 0 };
1942         list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1943         kmutex_t *hash_lock;
1944         uint64_t bytes_deleted = 0;
1945         uint64_t bufs_skipped = 0;
1946         int count = 0;
1947 
1948         ASSERT(GHOST_STATE(state));
1949 top:
1950         mutex_enter(&state->arcs_mtx);
1951         for (ab = list_tail(list); ab; ab = ab_prev) {
1952                 ab_prev = list_prev(list, ab);
1953                 if (ab->b_type > ARC_BUFC_NUMTYPES)
1954                         panic("invalid ab=%p", (void *)ab);
1955                 if (spa && ab->b_spa != spa)
1956                         continue;
1957 
1958                 /* ignore markers */
1959                 if (ab->b_spa == 0)
1960                         continue;
1961 
1962                 hash_lock = HDR_LOCK(ab);
1963                 /* caller may be trying to modify this buffer, skip it */
1964                 if (MUTEX_HELD(hash_lock))
1965                         continue;
1966 
1967                 /*
1968                  * It may take a long time to evict all the bufs requested.
1969                  * To avoid blocking all arc activity, periodically drop
1970                  * the arcs_mtx and give other threads a chance to run
1971                  * before reacquiring the lock.
1972                  */
1973                 if (count++ > arc_evict_iterations) {
1974                         list_insert_after(list, ab, &marker);
1975                         mutex_exit(&state->arcs_mtx);
1976                         kpreempt(KPREEMPT_SYNC);
1977                         mutex_enter(&state->arcs_mtx);
1978                         ab_prev = list_prev(list, &marker);
1979                         list_remove(list, &marker);
1980                         count = 0;
1981                         continue;
1982                 }
1983                 if (mutex_tryenter(hash_lock)) {
1984                         ASSERT(!HDR_IO_IN_PROGRESS(ab));
1985                         ASSERT(ab->b_buf == NULL);
1986                         ARCSTAT_BUMP(arcstat_deleted);
1987                         bytes_deleted += ab->b_size;
1988 
1989                         if (ab->b_l2hdr != NULL) {
1990                                 /*
1991                                  * This buffer is cached on the 2nd Level ARC;
1992                                  * don't destroy the header.
1993                                  */
1994                                 arc_change_state(arc_l2c_only, ab, hash_lock);
1995                                 mutex_exit(hash_lock);
1996                         } else {
1997                                 arc_change_state(arc_anon, ab, hash_lock);
1998                                 mutex_exit(hash_lock);
1999                                 arc_hdr_destroy(ab);
2000                         }
2001 
2002                         DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
2003                         if (bytes >= 0 && bytes_deleted >= bytes)
2004                                 break;
2005                 } else if (bytes < 0) {
2006                         /*
2007                          * Insert a list marker and then wait for the
2008                          * hash lock to become available. Once its
2009                          * available, restart from where we left off.
2010                          */
2011                         list_insert_after(list, ab, &marker);
2012                         mutex_exit(&state->arcs_mtx);
2013                         mutex_enter(hash_lock);
2014                         mutex_exit(hash_lock);
2015                         mutex_enter(&state->arcs_mtx);
2016                         ab_prev = list_prev(list, &marker);
2017                         list_remove(list, &marker);
2018                 } else {
2019                         bufs_skipped += 1;
2020                 }
2021 
2022         }
2023         mutex_exit(&state->arcs_mtx);
2024 
2025         if (list == &state->arcs_list[ARC_BUFC_DATA] &&
2026             (bytes < 0 || bytes_deleted < bytes)) {
2027                 list = &state->arcs_list[ARC_BUFC_METADATA];
2028                 goto top;
2029         }
2030 
2031         if (bufs_skipped) {
2032                 ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
2033                 ASSERT(bytes >= 0);
2034         }
2035 
2036         if (bytes_deleted < bytes)
2037                 dprintf("only deleted %lld bytes from %p",
2038                     (longlong_t)bytes_deleted, state);
2039 }
2040 
2041 static void
2042 arc_adjust(void)
2043 {
2044         int64_t adjustment, delta;
2045 
2046         /*
2047          * Adjust MRU size
2048          */
2049 
2050         adjustment = MIN((int64_t)(arc_size - arc_c),
2051             (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
2052             arc_p));
2053 
2054         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
2055                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
2056                 (void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
2057                 adjustment -= delta;
2058         }
2059 
2060         if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2061                 delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
2062                 (void) arc_evict(arc_mru, NULL, delta, FALSE,
2063                     ARC_BUFC_METADATA);
2064         }
2065 
2066         /*
2067          * Adjust MFU size
2068          */
2069 
2070         adjustment = arc_size - arc_c;
2071 
2072         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
2073                 delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
2074                 (void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
2075                 adjustment -= delta;
2076         }
2077 
2078         if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
2079                 int64_t delta = MIN(adjustment,
2080                     arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
2081                 (void) arc_evict(arc_mfu, NULL, delta, FALSE,
2082                     ARC_BUFC_METADATA);
2083         }
2084 
2085         /*
2086          * Adjust ghost lists
2087          */
2088 
2089         adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2090 
2091         if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
2092                 delta = MIN(arc_mru_ghost->arcs_size, adjustment);
2093                 arc_evict_ghost(arc_mru_ghost, NULL, delta);
2094         }
2095 
2096         adjustment =
2097             arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2098 
2099         if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
2100                 delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
2101                 arc_evict_ghost(arc_mfu_ghost, NULL, delta);
2102         }
2103 }
2104 
2105 static void
2106 arc_do_user_evicts(void)
2107 {
2108         mutex_enter(&arc_eviction_mtx);
2109         while (arc_eviction_list != NULL) {
2110                 arc_buf_t *buf = arc_eviction_list;
2111                 arc_eviction_list = buf->b_next;
2112                 mutex_enter(&buf->b_evict_lock);
2113                 buf->b_hdr = NULL;
2114                 mutex_exit(&buf->b_evict_lock);
2115                 mutex_exit(&arc_eviction_mtx);
2116 
2117                 if (buf->b_efunc != NULL)
2118                         VERIFY(buf->b_efunc(buf) == 0);
2119 
2120                 buf->b_efunc = NULL;
2121                 buf->b_private = NULL;
2122                 kmem_cache_free(buf_cache, buf);
2123                 mutex_enter(&arc_eviction_mtx);
2124         }
2125         mutex_exit(&arc_eviction_mtx);
2126 }
2127 
2128 /*
2129  * Flush all *evictable* data from the cache for the given spa.
2130  * NOTE: this will not touch "active" (i.e. referenced) data.
2131  */
2132 void
2133 arc_flush(spa_t *spa)
2134 {
2135         uint64_t guid = 0;
2136 
2137         if (spa)
2138                 guid = spa_load_guid(spa);
2139 
2140         while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
2141                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
2142                 if (spa)
2143                         break;
2144         }
2145         while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
2146                 (void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
2147                 if (spa)
2148                         break;
2149         }
2150         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
2151                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
2152                 if (spa)
2153                         break;
2154         }
2155         while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
2156                 (void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
2157                 if (spa)
2158                         break;
2159         }
2160 
2161         arc_evict_ghost(arc_mru_ghost, guid, -1);
2162         arc_evict_ghost(arc_mfu_ghost, guid, -1);
2163 
2164         mutex_enter(&arc_reclaim_thr_lock);
2165         arc_do_user_evicts();
2166         mutex_exit(&arc_reclaim_thr_lock);
2167         ASSERT(spa || arc_eviction_list == NULL);
2168 }
2169 
2170 void
2171 arc_shrink(void)
2172 {
2173         if (arc_c > arc_c_min) {
2174                 uint64_t to_free;
2175 
2176 #ifdef _KERNEL
2177                 to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
2178 #else
2179                 to_free = arc_c >> arc_shrink_shift;
2180 #endif
2181                 if (arc_c > arc_c_min + to_free)
2182                         atomic_add_64(&arc_c, -to_free);
2183                 else
2184                         arc_c = arc_c_min;
2185 
2186                 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
2187                 if (arc_c > arc_size)
2188                         arc_c = MAX(arc_size, arc_c_min);
2189                 if (arc_p > arc_c)
2190                         arc_p = (arc_c >> 1);
2191                 ASSERT(arc_c >= arc_c_min);
2192                 ASSERT((int64_t)arc_p >= 0);
2193         }
2194 
2195         if (arc_size > arc_c)
2196                 arc_adjust();
2197 }
2198 
2199 /*
2200  * Determine if the system is under memory pressure and is asking
2201  * to reclaim memory. A return value of 1 indicates that the system
2202  * is under memory pressure and that the arc should adjust accordingly.
2203  */
2204 static int
2205 arc_reclaim_needed(void)
2206 {
2207         uint64_t extra;
2208 
2209 #ifdef _KERNEL
2210 
2211         if (needfree)
2212                 return (1);
2213 
2214         /*
2215          * take 'desfree' extra pages, so we reclaim sooner, rather than later
2216          */
2217         extra = desfree;
2218 
2219         /*
2220          * check that we're out of range of the pageout scanner.  It starts to
2221          * schedule paging if freemem is less than lotsfree and needfree.
2222          * lotsfree is the high-water mark for pageout, and needfree is the
2223          * number of needed free pages.  We add extra pages here to make sure
2224          * the scanner doesn't start up while we're freeing memory.
2225          */
2226         if (freemem < lotsfree + needfree + extra)
2227                 return (1);
2228 
2229         /*
2230          * check to make sure that swapfs has enough space so that anon
2231          * reservations can still succeed. anon_resvmem() checks that the
2232          * availrmem is greater than swapfs_minfree, and the number of reserved
2233          * swap pages.  We also add a bit of extra here just to prevent
2234          * circumstances from getting really dire.
2235          */
2236         if (availrmem < swapfs_minfree + swapfs_reserve + extra)
2237                 return (1);
2238 
2239         /*
2240          * Check that we have enough availrmem that memory locking (e.g., via
2241          * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
2242          * stores the number of pages that cannot be locked; when availrmem
2243          * drops below pages_pp_maximum, page locking mechanisms such as
2244          * page_pp_lock() will fail.)
2245          */
2246         if (availrmem <= pages_pp_maximum)
2247                 return (1);
2248 
2249 #if defined(__i386)
2250         /*
2251          * If we're on an i386 platform, it's possible that we'll exhaust the
2252          * kernel heap space before we ever run out of available physical
2253          * memory.  Most checks of the size of the heap_area compare against
2254          * tune.t_minarmem, which is the minimum available real memory that we
2255          * can have in the system.  However, this is generally fixed at 25 pages
2256          * which is so low that it's useless.  In this comparison, we seek to
2257          * calculate the total heap-size, and reclaim if more than 3/4ths of the
2258          * heap is allocated.  (Or, in the calculation, if less than 1/4th is
2259          * free)
2260          */
2261         if (vmem_size(heap_arena, VMEM_FREE) <
2262             (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
2263                 return (1);
2264 #endif
2265 
2266         /*
2267          * If zio data pages are being allocated out of a separate heap segment,
2268          * then enforce that the size of available vmem for this arena remains
2269          * above about 1/16th free.
2270          *
2271          * Note: The 1/16th arena free requirement was put in place
2272          * to aggressively evict memory from the arc in order to avoid
2273          * memory fragmentation issues.
2274          */
2275         if (zio_arena != NULL &&
2276             vmem_size(zio_arena, VMEM_FREE) <
2277             (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
2278                 return (1);
2279 #else
2280         if (spa_get_random(100) == 0)
2281                 return (1);
2282 #endif
2283         return (0);
2284 }
2285 
2286 static void
2287 arc_kmem_reap_now(arc_reclaim_strategy_t strat)
2288 {
2289         size_t                  i;
2290         kmem_cache_t            *prev_cache = NULL;
2291         kmem_cache_t            *prev_data_cache = NULL;
2292         extern kmem_cache_t     *zio_buf_cache[];
2293         extern kmem_cache_t     *zio_data_buf_cache[];
2294 
2295 #ifdef _KERNEL
2296         if (arc_meta_used >= arc_meta_limit) {
2297                 /*
2298                  * We are exceeding our meta-data cache limit.
2299                  * Purge some DNLC entries to release holds on meta-data.
2300                  */
2301                 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
2302         }
2303 #if defined(__i386)
2304         /*
2305          * Reclaim unused memory from all kmem caches.
2306          */
2307         kmem_reap();
2308 #endif
2309 #endif
2310 
2311         /*
2312          * An aggressive reclamation will shrink the cache size as well as
2313          * reap free buffers from the arc kmem caches.
2314          */
2315         if (strat == ARC_RECLAIM_AGGR)
2316                 arc_shrink();
2317 
2318         for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
2319                 if (zio_buf_cache[i] != prev_cache) {
2320                         prev_cache = zio_buf_cache[i];
2321                         kmem_cache_reap_now(zio_buf_cache[i]);
2322                 }
2323                 if (zio_data_buf_cache[i] != prev_data_cache) {
2324                         prev_data_cache = zio_data_buf_cache[i];
2325                         kmem_cache_reap_now(zio_data_buf_cache[i]);
2326                 }
2327         }
2328         kmem_cache_reap_now(buf_cache);
2329         kmem_cache_reap_now(hdr_cache);
2330 
2331         /*
2332          * Ask the vmem areana to reclaim unused memory from its
2333          * quantum caches.
2334          */
2335         if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
2336                 vmem_qcache_reap(zio_arena);
2337 }
2338 
2339 static void
2340 arc_reclaim_thread(void)
2341 {
2342         clock_t                 growtime = 0;
2343         arc_reclaim_strategy_t  last_reclaim = ARC_RECLAIM_CONS;
2344         callb_cpr_t             cpr;
2345 
2346         CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
2347 
2348         mutex_enter(&arc_reclaim_thr_lock);
2349         while (arc_thread_exit == 0) {
2350                 if (arc_reclaim_needed()) {
2351 
2352                         if (arc_no_grow) {
2353                                 if (last_reclaim == ARC_RECLAIM_CONS) {
2354                                         last_reclaim = ARC_RECLAIM_AGGR;
2355                                 } else {
2356                                         last_reclaim = ARC_RECLAIM_CONS;
2357                                 }
2358                         } else {
2359                                 arc_no_grow = TRUE;
2360                                 last_reclaim = ARC_RECLAIM_AGGR;
2361                                 membar_producer();
2362                         }
2363 
2364                         /* reset the growth delay for every reclaim */
2365                         growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
2366 
2367                         arc_kmem_reap_now(last_reclaim);
2368                         arc_warm = B_TRUE;
2369 
2370                 } else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
2371                         arc_no_grow = FALSE;
2372                 }
2373 
2374                 arc_adjust();
2375 
2376                 if (arc_eviction_list != NULL)
2377                         arc_do_user_evicts();
2378 
2379                 /* block until needed, or one second, whichever is shorter */
2380                 CALLB_CPR_SAFE_BEGIN(&cpr);
2381                 (void) cv_timedwait(&arc_reclaim_thr_cv,
2382                     &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
2383                 CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
2384         }
2385 
2386         arc_thread_exit = 0;
2387         cv_broadcast(&arc_reclaim_thr_cv);
2388         CALLB_CPR_EXIT(&cpr);               /* drops arc_reclaim_thr_lock */
2389         thread_exit();
2390 }
2391 
2392 /*
2393  * Adapt arc info given the number of bytes we are trying to add and
2394  * the state that we are comming from.  This function is only called
2395  * when we are adding new content to the cache.
2396  */
2397 static void
2398 arc_adapt(int bytes, arc_state_t *state)
2399 {
2400         int mult;
2401         uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
2402 
2403         if (state == arc_l2c_only)
2404                 return;
2405 
2406         ASSERT(bytes > 0);
2407         /*
2408          * Adapt the target size of the MRU list:
2409          *      - if we just hit in the MRU ghost list, then increase
2410          *        the target size of the MRU list.
2411          *      - if we just hit in the MFU ghost list, then increase
2412          *        the target size of the MFU list by decreasing the
2413          *        target size of the MRU list.
2414          */
2415         if (state == arc_mru_ghost) {
2416                 mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
2417                     1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
2418                 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
2419 
2420                 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
2421         } else if (state == arc_mfu_ghost) {
2422                 uint64_t delta;
2423 
2424                 mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
2425                     1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
2426                 mult = MIN(mult, 10);
2427 
2428                 delta = MIN(bytes * mult, arc_p);
2429                 arc_p = MAX(arc_p_min, arc_p - delta);
2430         }
2431         ASSERT((int64_t)arc_p >= 0);
2432 
2433         if (arc_reclaim_needed()) {
2434                 cv_signal(&arc_reclaim_thr_cv);
2435                 return;
2436         }
2437 
2438         if (arc_no_grow)
2439                 return;
2440 
2441         if (arc_c >= arc_c_max)
2442                 return;
2443 
2444         /*
2445          * If we're within (2 * maxblocksize) bytes of the target
2446          * cache size, increment the target cache size
2447          */
2448         if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
2449                 atomic_add_64(&arc_c, (int64_t)bytes);
2450                 if (arc_c > arc_c_max)
2451                         arc_c = arc_c_max;
2452                 else if (state == arc_anon)
2453                         atomic_add_64(&arc_p, (int64_t)bytes);
2454                 if (arc_p > arc_c)
2455                         arc_p = arc_c;
2456         }
2457         ASSERT((int64_t)arc_p >= 0);
2458 }
2459 
2460 /*
2461  * Check if the cache has reached its limits and eviction is required
2462  * prior to insert.
2463  */
2464 static int
2465 arc_evict_needed(arc_buf_contents_t type)
2466 {
2467         if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
2468                 return (1);
2469 
2470         if (arc_reclaim_needed())
2471                 return (1);
2472 
2473         return (arc_size > arc_c);
2474 }
2475 
2476 /*
2477  * The buffer, supplied as the first argument, needs a data block.
2478  * So, if we are at cache max, determine which cache should be victimized.
2479  * We have the following cases:
2480  *
2481  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
2482  * In this situation if we're out of space, but the resident size of the MFU is
2483  * under the limit, victimize the MFU cache to satisfy this insertion request.
2484  *
2485  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
2486  * Here, we've used up all of the available space for the MRU, so we need to
2487  * evict from our own cache instead.  Evict from the set of resident MRU
2488  * entries.
2489  *
2490  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
2491  * c minus p represents the MFU space in the cache, since p is the size of the
2492  * cache that is dedicated to the MRU.  In this situation there's still space on
2493  * the MFU side, so the MRU side needs to be victimized.
2494  *
2495  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
2496  * MFU's resident set is consuming more space than it has been allotted.  In
2497  * this situation, we must victimize our own cache, the MFU, for this insertion.
2498  */
2499 static void
2500 arc_get_data_buf(arc_buf_t *buf)
2501 {
2502         arc_state_t             *state = buf->b_hdr->b_state;
2503         uint64_t                size = buf->b_hdr->b_size;
2504         arc_buf_contents_t      type = buf->b_hdr->b_type;
2505 
2506         arc_adapt(size, state);
2507 
2508         /*
2509          * We have not yet reached cache maximum size,
2510          * just allocate a new buffer.
2511          */
2512         if (!arc_evict_needed(type)) {
2513                 if (type == ARC_BUFC_METADATA) {
2514                         buf->b_data = zio_buf_alloc(size);
2515                         arc_space_consume(size, ARC_SPACE_DATA);
2516                 } else {
2517                         ASSERT(type == ARC_BUFC_DATA);
2518                         buf->b_data = zio_data_buf_alloc(size);
2519                         ARCSTAT_INCR(arcstat_data_size, size);
2520                         atomic_add_64(&arc_size, size);
2521                 }
2522                 goto out;
2523         }
2524 
2525         /*
2526          * If we are prefetching from the mfu ghost list, this buffer
2527          * will end up on the mru list; so steal space from there.
2528          */
2529         if (state == arc_mfu_ghost)
2530                 state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
2531         else if (state == arc_mru_ghost)
2532                 state = arc_mru;
2533 
2534         if (state == arc_mru || state == arc_anon) {
2535                 uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
2536                 state = (arc_mfu->arcs_lsize[type] >= size &&
2537                     arc_p > mru_used) ? arc_mfu : arc_mru;
2538         } else {
2539                 /* MFU cases */
2540                 uint64_t mfu_space = arc_c - arc_p;
2541                 state =  (arc_mru->arcs_lsize[type] >= size &&
2542                     mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
2543         }
2544         if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
2545                 if (type == ARC_BUFC_METADATA) {
2546                         buf->b_data = zio_buf_alloc(size);
2547                         arc_space_consume(size, ARC_SPACE_DATA);
2548                 } else {
2549                         ASSERT(type == ARC_BUFC_DATA);
2550                         buf->b_data = zio_data_buf_alloc(size);
2551                         ARCSTAT_INCR(arcstat_data_size, size);
2552                         atomic_add_64(&arc_size, size);
2553                 }
2554                 ARCSTAT_BUMP(arcstat_recycle_miss);
2555         }
2556         ASSERT(buf->b_data != NULL);
2557 out:
2558         /*
2559          * Update the state size.  Note that ghost states have a
2560          * "ghost size" and so don't need to be updated.
2561          */
2562         if (!GHOST_STATE(buf->b_hdr->b_state)) {
2563                 arc_buf_hdr_t *hdr = buf->b_hdr;
2564 
2565                 atomic_add_64(&hdr->b_state->arcs_size, size);
2566                 if (list_link_active(&hdr->b_arc_node)) {
2567                         ASSERT(refcount_is_zero(&hdr->b_refcnt));
2568                         atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
2569                 }
2570                 /*
2571                  * If we are growing the cache, and we are adding anonymous
2572                  * data, and we have outgrown arc_p, update arc_p
2573                  */
2574                 if (arc_size < arc_c && hdr->b_state == arc_anon &&
2575                     arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
2576                         arc_p = MIN(arc_c, arc_p + size);
2577         }
2578 }
2579 
2580 /*
2581  * This routine is called whenever a buffer is accessed.
2582  * NOTE: the hash lock is dropped in this function.
2583  */
2584 static void
2585 arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
2586 {
2587         clock_t now;
2588 
2589         ASSERT(MUTEX_HELD(hash_lock));
2590 
2591         if (buf->b_state == arc_anon) {
2592                 /*
2593                  * This buffer is not in the cache, and does not
2594                  * appear in our "ghost" list.  Add the new buffer
2595                  * to the MRU state.
2596                  */
2597 
2598                 ASSERT(buf->b_arc_access == 0);
2599                 buf->b_arc_access = ddi_get_lbolt();
2600                 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2601                 arc_change_state(arc_mru, buf, hash_lock);
2602 
2603         } else if (buf->b_state == arc_mru) {
2604                 now = ddi_get_lbolt();
2605 
2606                 /*
2607                  * If this buffer is here because of a prefetch, then either:
2608                  * - clear the flag if this is a "referencing" read
2609                  *   (any subsequent access will bump this into the MFU state).
2610                  * or
2611                  * - move the buffer to the head of the list if this is
2612                  *   another prefetch (to make it less likely to be evicted).
2613                  */
2614                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2615                         if (refcount_count(&buf->b_refcnt) == 0) {
2616                                 ASSERT(list_link_active(&buf->b_arc_node));
2617                         } else {
2618                                 buf->b_flags &= ~ARC_PREFETCH;
2619                                 ARCSTAT_BUMP(arcstat_mru_hits);
2620                         }
2621                         buf->b_arc_access = now;
2622                         return;
2623                 }
2624 
2625                 /*
2626                  * This buffer has been "accessed" only once so far,
2627                  * but it is still in the cache. Move it to the MFU
2628                  * state.
2629                  */
2630                 if (now > buf->b_arc_access + ARC_MINTIME) {
2631                         /*
2632                          * More than 125ms have passed since we
2633                          * instantiated this buffer.  Move it to the
2634                          * most frequently used state.
2635                          */
2636                         buf->b_arc_access = now;
2637                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2638                         arc_change_state(arc_mfu, buf, hash_lock);
2639                 }
2640                 ARCSTAT_BUMP(arcstat_mru_hits);
2641         } else if (buf->b_state == arc_mru_ghost) {
2642                 arc_state_t     *new_state;
2643                 /*
2644                  * This buffer has been "accessed" recently, but
2645                  * was evicted from the cache.  Move it to the
2646                  * MFU state.
2647                  */
2648 
2649                 if (buf->b_flags & ARC_PREFETCH) {
2650                         new_state = arc_mru;
2651                         if (refcount_count(&buf->b_refcnt) > 0)
2652                                 buf->b_flags &= ~ARC_PREFETCH;
2653                         DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
2654                 } else {
2655                         new_state = arc_mfu;
2656                         DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2657                 }
2658 
2659                 buf->b_arc_access = ddi_get_lbolt();
2660                 arc_change_state(new_state, buf, hash_lock);
2661 
2662                 ARCSTAT_BUMP(arcstat_mru_ghost_hits);
2663         } else if (buf->b_state == arc_mfu) {
2664                 /*
2665                  * This buffer has been accessed more than once and is
2666                  * still in the cache.  Keep it in the MFU state.
2667                  *
2668                  * NOTE: an add_reference() that occurred when we did
2669                  * the arc_read() will have kicked this off the list.
2670                  * If it was a prefetch, we will explicitly move it to
2671                  * the head of the list now.
2672                  */
2673                 if ((buf->b_flags & ARC_PREFETCH) != 0) {
2674                         ASSERT(refcount_count(&buf->b_refcnt) == 0);
2675                         ASSERT(list_link_active(&buf->b_arc_node));
2676                 }
2677                 ARCSTAT_BUMP(arcstat_mfu_hits);
2678                 buf->b_arc_access = ddi_get_lbolt();
2679         } else if (buf->b_state == arc_mfu_ghost) {
2680                 arc_state_t     *new_state = arc_mfu;
2681                 /*
2682                  * This buffer has been accessed more than once but has
2683                  * been evicted from the cache.  Move it back to the
2684                  * MFU state.
2685                  */
2686 
2687                 if (buf->b_flags & ARC_PREFETCH) {
2688                         /*
2689                          * This is a prefetch access...
2690                          * move this block back to the MRU state.
2691                          */
2692                         ASSERT0(refcount_count(&buf->b_refcnt));
2693                         new_state = arc_mru;
2694                 }
2695 
2696                 buf->b_arc_access = ddi_get_lbolt();
2697                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2698                 arc_change_state(new_state, buf, hash_lock);
2699 
2700                 ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
2701         } else if (buf->b_state == arc_l2c_only) {
2702                 /*
2703                  * This buffer is on the 2nd Level ARC.
2704                  */
2705 
2706                 buf->b_arc_access = ddi_get_lbolt();
2707                 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
2708                 arc_change_state(arc_mfu, buf, hash_lock);
2709         } else {
2710                 ASSERT(!"invalid arc state");
2711         }
2712 }
2713 
2714 /* a generic arc_done_func_t which you can use */
2715 /* ARGSUSED */
2716 void
2717 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
2718 {
2719         if (zio == NULL || zio->io_error == 0)
2720                 bcopy(buf->b_data, arg, buf->b_hdr->b_size);
2721         VERIFY(arc_buf_remove_ref(buf, arg));
2722 }
2723 
2724 /* a generic arc_done_func_t */
2725 void
2726 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
2727 {
2728         arc_buf_t **bufp = arg;
2729         if (zio && zio->io_error) {
2730                 VERIFY(arc_buf_remove_ref(buf, arg));
2731                 *bufp = NULL;
2732         } else {
2733                 *bufp = buf;
2734                 ASSERT(buf->b_data);
2735         }
2736 }
2737 
2738 static void
2739 arc_read_done(zio_t *zio)
2740 {
2741         arc_buf_hdr_t   *hdr, *found;
2742         arc_buf_t       *buf;
2743         arc_buf_t       *abuf;  /* buffer we're assigning to callback */
2744         kmutex_t        *hash_lock;
2745         arc_callback_t  *callback_list, *acb;
2746         int             freeable = FALSE;
2747 
2748         buf = zio->io_private;
2749         hdr = buf->b_hdr;
2750 
2751         /*
2752          * The hdr was inserted into hash-table and removed from lists
2753          * prior to starting I/O.  We should find this header, since
2754          * it's in the hash table, and it should be legit since it's
2755          * not possible to evict it during the I/O.  The only possible
2756          * reason for it not to be found is if we were freed during the
2757          * read.
2758          */
2759         found = buf_hash_find(hdr->b_spa, &hdr->b_dva, hdr->b_birth,
2760             &hash_lock);
2761 
2762         ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
2763             (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
2764             (found == hdr && HDR_L2_READING(hdr)));
2765 
2766         hdr->b_flags &= ~ARC_L2_EVICTED;
2767         if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
2768                 hdr->b_flags &= ~ARC_L2CACHE;
2769 
2770         /* byteswap if necessary */
2771         callback_list = hdr->b_acb;
2772         ASSERT(callback_list != NULL);
2773         if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
2774                 dmu_object_byteswap_t bswap =
2775                     DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
2776                 arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
2777                     byteswap_uint64_array :
2778                     dmu_ot_byteswap[bswap].ob_func;
2779                 func(buf->b_data, hdr->b_size);
2780         }
2781 
2782         arc_cksum_compute(buf, B_FALSE);
2783         arc_buf_watch(buf);
2784 
2785         if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
2786                 /*
2787                  * Only call arc_access on anonymous buffers.  This is because
2788                  * if we've issued an I/O for an evicted buffer, we've already
2789                  * called arc_access (to prevent any simultaneous readers from
2790                  * getting confused).
2791                  */
2792                 arc_access(hdr, hash_lock);
2793         }
2794 
2795         /* create copies of the data buffer for the callers */
2796         abuf = buf;
2797         for (acb = callback_list; acb; acb = acb->acb_next) {
2798                 if (acb->acb_done) {
2799                         if (abuf == NULL) {
2800                                 ARCSTAT_BUMP(arcstat_duplicate_reads);
2801                                 abuf = arc_buf_clone(buf);
2802                         }
2803                         acb->acb_buf = abuf;
2804                         abuf = NULL;
2805                 }
2806         }
2807         hdr->b_acb = NULL;
2808         hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2809         ASSERT(!HDR_BUF_AVAILABLE(hdr));
2810         if (abuf == buf) {
2811                 ASSERT(buf->b_efunc == NULL);
2812                 ASSERT(hdr->b_datacnt == 1);
2813                 hdr->b_flags |= ARC_BUF_AVAILABLE;
2814         }
2815 
2816         ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2817 
2818         if (zio->io_error != 0) {
2819                 hdr->b_flags |= ARC_IO_ERROR;
2820                 if (hdr->b_state != arc_anon)
2821                         arc_change_state(arc_anon, hdr, hash_lock);
2822                 if (HDR_IN_HASH_TABLE(hdr))
2823                         buf_hash_remove(hdr);
2824                 freeable = refcount_is_zero(&hdr->b_refcnt);
2825         }
2826 
2827         /*
2828          * Broadcast before we drop the hash_lock to avoid the possibility
2829          * that the hdr (and hence the cv) might be freed before we get to
2830          * the cv_broadcast().
2831          */
2832         cv_broadcast(&hdr->b_cv);
2833 
2834         if (hash_lock) {
2835                 mutex_exit(hash_lock);
2836         } else {
2837                 /*
2838                  * This block was freed while we waited for the read to
2839                  * complete.  It has been removed from the hash table and
2840                  * moved to the anonymous state (so that it won't show up
2841                  * in the cache).
2842                  */
2843                 ASSERT3P(hdr->b_state, ==, arc_anon);
2844                 freeable = refcount_is_zero(&hdr->b_refcnt);
2845         }
2846 
2847         /* execute each callback and free its structure */
2848         while ((acb = callback_list) != NULL) {
2849                 if (acb->acb_done)
2850                         acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2851 
2852                 if (acb->acb_zio_dummy != NULL) {
2853                         acb->acb_zio_dummy->io_error = zio->io_error;
2854                         zio_nowait(acb->acb_zio_dummy);
2855                 }
2856 
2857                 callback_list = acb->acb_next;
2858                 kmem_free(acb, sizeof (arc_callback_t));
2859         }
2860 
2861         if (freeable)
2862                 arc_hdr_destroy(hdr);
2863 }
2864 
2865 /*
2866  * "Read" the block at the specified DVA (in bp) via the
2867  * cache.  If the block is found in the cache, invoke the provided
2868  * callback immediately and return.  Note that the `zio' parameter
2869  * in the callback will be NULL in this case, since no IO was
2870  * required.  If the block is not in the cache pass the read request
2871  * on to the spa with a substitute callback function, so that the
2872  * requested block will be added to the cache.
2873  *
2874  * If a read request arrives for a block that has a read in-progress,
2875  * either wait for the in-progress read to complete (and return the
2876  * results); or, if this is a read with a "done" func, add a record
2877  * to the read to invoke the "done" func when the read completes,
2878  * and return; or just return.
2879  *
2880  * arc_read_done() will invoke all the requested "done" functions
2881  * for readers of this block.
2882  */
2883 int
2884 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
2885     void *private, zio_priority_t priority, int zio_flags, uint32_t *arc_flags,
2886     const zbookmark_t *zb)
2887 {
2888         arc_buf_hdr_t *hdr;
2889         arc_buf_t *buf = NULL;
2890         kmutex_t *hash_lock;
2891         zio_t *rzio;
2892         uint64_t guid = spa_load_guid(spa);
2893 
2894 top:
2895         hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
2896             &hash_lock);
2897         if (hdr && hdr->b_datacnt > 0) {
2898 
2899                 *arc_flags |= ARC_CACHED;
2900 
2901                 if (HDR_IO_IN_PROGRESS(hdr)) {
2902 
2903                         if (*arc_flags & ARC_WAIT) {
2904                                 cv_wait(&hdr->b_cv, hash_lock);
2905                                 mutex_exit(hash_lock);
2906                                 goto top;
2907                         }
2908                         ASSERT(*arc_flags & ARC_NOWAIT);
2909 
2910                         if (done) {
2911                                 arc_callback_t  *acb = NULL;
2912 
2913                                 acb = kmem_zalloc(sizeof (arc_callback_t),
2914                                     KM_SLEEP);
2915                                 acb->acb_done = done;
2916                                 acb->acb_private = private;
2917                                 if (pio != NULL)
2918                                         acb->acb_zio_dummy = zio_null(pio,
2919                                             spa, NULL, NULL, NULL, zio_flags);
2920 
2921                                 ASSERT(acb->acb_done != NULL);
2922                                 acb->acb_next = hdr->b_acb;
2923                                 hdr->b_acb = acb;
2924                                 add_reference(hdr, hash_lock, private);
2925                                 mutex_exit(hash_lock);
2926                                 return (0);
2927                         }
2928                         mutex_exit(hash_lock);
2929                         return (0);
2930                 }
2931 
2932                 ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2933 
2934                 if (done) {
2935                         add_reference(hdr, hash_lock, private);
2936                         /*
2937                          * If this block is already in use, create a new
2938                          * copy of the data so that we will be guaranteed
2939                          * that arc_release() will always succeed.
2940                          */
2941                         buf = hdr->b_buf;
2942                         ASSERT(buf);
2943                         ASSERT(buf->b_data);
2944                         if (HDR_BUF_AVAILABLE(hdr)) {
2945                                 ASSERT(buf->b_efunc == NULL);
2946                                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
2947                         } else {
2948                                 buf = arc_buf_clone(buf);
2949                         }
2950 
2951                 } else if (*arc_flags & ARC_PREFETCH &&
2952                     refcount_count(&hdr->b_refcnt) == 0) {
2953                         hdr->b_flags |= ARC_PREFETCH;
2954                 }
2955                 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
2956                 arc_access(hdr, hash_lock);
2957                 if (*arc_flags & ARC_L2CACHE)
2958                         hdr->b_flags |= ARC_L2CACHE;
2959                 if (*arc_flags & ARC_L2COMPRESS)
2960                         hdr->b_flags |= ARC_L2COMPRESS;
2961                 mutex_exit(hash_lock);
2962                 ARCSTAT_BUMP(arcstat_hits);
2963                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
2964                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
2965                     data, metadata, hits);
2966 
2967                 if (done)
2968                         done(NULL, buf, private);
2969         } else {
2970                 uint64_t size = BP_GET_LSIZE(bp);
2971                 arc_callback_t  *acb;
2972                 vdev_t *vd = NULL;
2973                 uint64_t addr = 0;
2974                 boolean_t devw = B_FALSE;
2975                 enum zio_compress b_compress = ZIO_COMPRESS_OFF;
2976                 uint64_t b_asize = 0;
2977 
2978                 if (hdr == NULL) {
2979                         /* this block is not in the cache */
2980                         arc_buf_hdr_t   *exists;
2981                         arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
2982                         buf = arc_buf_alloc(spa, size, private, type);
2983                         hdr = buf->b_hdr;
2984                         hdr->b_dva = *BP_IDENTITY(bp);
2985                         hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
2986                         hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2987                         exists = buf_hash_insert(hdr, &hash_lock);
2988                         if (exists) {
2989                                 /* somebody beat us to the hash insert */
2990                                 mutex_exit(hash_lock);
2991                                 buf_discard_identity(hdr);
2992                                 (void) arc_buf_remove_ref(buf, private);
2993                                 goto top; /* restart the IO request */
2994                         }
2995                         /* if this is a prefetch, we don't have a reference */
2996                         if (*arc_flags & ARC_PREFETCH) {
2997                                 (void) remove_reference(hdr, hash_lock,
2998                                     private);
2999                                 hdr->b_flags |= ARC_PREFETCH;
3000                         }
3001                         if (*arc_flags & ARC_L2CACHE)
3002                                 hdr->b_flags |= ARC_L2CACHE;
3003                         if (*arc_flags & ARC_L2COMPRESS)
3004                                 hdr->b_flags |= ARC_L2COMPRESS;
3005                         if (BP_GET_LEVEL(bp) > 0)
3006                                 hdr->b_flags |= ARC_INDIRECT;
3007                 } else {
3008                         /* this block is in the ghost cache */
3009                         ASSERT(GHOST_STATE(hdr->b_state));
3010                         ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3011                         ASSERT0(refcount_count(&hdr->b_refcnt));
3012                         ASSERT(hdr->b_buf == NULL);
3013 
3014                         /* if this is a prefetch, we don't have a reference */
3015                         if (*arc_flags & ARC_PREFETCH)
3016                                 hdr->b_flags |= ARC_PREFETCH;
3017                         else
3018                                 add_reference(hdr, hash_lock, private);
3019                         if (*arc_flags & ARC_L2CACHE)
3020                                 hdr->b_flags |= ARC_L2CACHE;
3021                         if (*arc_flags & ARC_L2COMPRESS)
3022                                 hdr->b_flags |= ARC_L2COMPRESS;
3023                         buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
3024                         buf->b_hdr = hdr;
3025                         buf->b_data = NULL;
3026                         buf->b_efunc = NULL;
3027                         buf->b_private = NULL;
3028                         buf->b_next = NULL;
3029                         hdr->b_buf = buf;
3030                         ASSERT(hdr->b_datacnt == 0);
3031                         hdr->b_datacnt = 1;
3032                         arc_get_data_buf(buf);
3033                         arc_access(hdr, hash_lock);
3034                 }
3035 
3036                 ASSERT(!GHOST_STATE(hdr->b_state));
3037 
3038                 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
3039                 acb->acb_done = done;
3040                 acb->acb_private = private;
3041 
3042                 ASSERT(hdr->b_acb == NULL);
3043                 hdr->b_acb = acb;
3044                 hdr->b_flags |= ARC_IO_IN_PROGRESS;
3045 
3046                 if (hdr->b_l2hdr != NULL &&
3047                     (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
3048                         devw = hdr->b_l2hdr->b_dev->l2ad_writing;
3049                         addr = hdr->b_l2hdr->b_daddr;
3050                         b_compress = hdr->b_l2hdr->b_compress;
3051                         b_asize = hdr->b_l2hdr->b_asize;
3052                         /*
3053                          * Lock out device removal.
3054                          */
3055                         if (vdev_is_dead(vd) ||
3056                             !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
3057                                 vd = NULL;
3058                 }
3059 
3060                 mutex_exit(hash_lock);
3061 
3062                 /*
3063                  * At this point, we have a level 1 cache miss.  Try again in
3064                  * L2ARC if possible.
3065                  */
3066                 ASSERT3U(hdr->b_size, ==, size);
3067                 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
3068                     uint64_t, size, zbookmark_t *, zb);
3069                 ARCSTAT_BUMP(arcstat_misses);
3070                 ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
3071                     demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
3072                     data, metadata, misses);
3073 
3074                 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
3075                         /*
3076                          * Read from the L2ARC if the following are true:
3077                          * 1. The L2ARC vdev was previously cached.
3078                          * 2. This buffer still has L2ARC metadata.
3079                          * 3. This buffer isn't currently writing to the L2ARC.
3080                          * 4. The L2ARC entry wasn't evicted, which may
3081                          *    also have invalidated the vdev.
3082                          * 5. This isn't prefetch and l2arc_noprefetch is set.
3083                          */
3084                         if (hdr->b_l2hdr != NULL &&
3085                             !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
3086                             !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
3087                                 l2arc_read_callback_t *cb;
3088 
3089                                 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
3090                                 ARCSTAT_BUMP(arcstat_l2_hits);
3091 
3092                                 cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
3093                                     KM_SLEEP);
3094                                 cb->l2rcb_buf = buf;
3095                                 cb->l2rcb_spa = spa;
3096                                 cb->l2rcb_bp = *bp;
3097                                 cb->l2rcb_zb = *zb;
3098                                 cb->l2rcb_flags = zio_flags;
3099                                 cb->l2rcb_compress = b_compress;
3100 
3101                                 ASSERT(addr >= VDEV_LABEL_START_SIZE &&
3102                                     addr + size < vd->vdev_psize -
3103                                     VDEV_LABEL_END_SIZE);
3104 
3105                                 /*
3106                                  * l2arc read.  The SCL_L2ARC lock will be
3107                                  * released by l2arc_read_done().
3108                                  * Issue a null zio if the underlying buffer
3109                                  * was squashed to zero size by compression.
3110                                  */
3111                                 if (b_compress == ZIO_COMPRESS_EMPTY) {
3112                                         rzio = zio_null(pio, spa, vd,
3113                                             l2arc_read_done, cb,
3114                                             zio_flags | ZIO_FLAG_DONT_CACHE |
3115                                             ZIO_FLAG_CANFAIL |
3116                                             ZIO_FLAG_DONT_PROPAGATE |
3117                                             ZIO_FLAG_DONT_RETRY);
3118                                 } else {
3119                                         rzio = zio_read_phys(pio, vd, addr,
3120                                             b_asize, buf->b_data,
3121                                             ZIO_CHECKSUM_OFF,
3122                                             l2arc_read_done, cb, priority,
3123                                             zio_flags | ZIO_FLAG_DONT_CACHE |
3124                                             ZIO_FLAG_CANFAIL |
3125                                             ZIO_FLAG_DONT_PROPAGATE |
3126                                             ZIO_FLAG_DONT_RETRY, B_FALSE);
3127                                 }
3128                                 DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
3129                                     zio_t *, rzio);
3130                                 ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
3131 
3132                                 if (*arc_flags & ARC_NOWAIT) {
3133                                         zio_nowait(rzio);
3134                                         return (0);
3135                                 }
3136 
3137                                 ASSERT(*arc_flags & ARC_WAIT);
3138                                 if (zio_wait(rzio) == 0)
3139                                         return (0);
3140 
3141                                 /* l2arc read error; goto zio_read() */
3142                         } else {
3143                                 DTRACE_PROBE1(l2arc__miss,
3144                                     arc_buf_hdr_t *, hdr);
3145                                 ARCSTAT_BUMP(arcstat_l2_misses);
3146                                 if (HDR_L2_WRITING(hdr))
3147                                         ARCSTAT_BUMP(arcstat_l2_rw_clash);
3148                                 spa_config_exit(spa, SCL_L2ARC, vd);
3149                         }
3150                 } else {
3151                         if (vd != NULL)
3152                                 spa_config_exit(spa, SCL_L2ARC, vd);
3153                         if (l2arc_ndev != 0) {
3154                                 DTRACE_PROBE1(l2arc__miss,
3155                                     arc_buf_hdr_t *, hdr);
3156                                 ARCSTAT_BUMP(arcstat_l2_misses);
3157                         }
3158                 }
3159 
3160                 rzio = zio_read(pio, spa, bp, buf->b_data, size,
3161                     arc_read_done, buf, priority, zio_flags, zb);
3162 
3163                 if (*arc_flags & ARC_WAIT)
3164                         return (zio_wait(rzio));
3165 
3166                 ASSERT(*arc_flags & ARC_NOWAIT);
3167                 zio_nowait(rzio);
3168         }
3169         return (0);
3170 }
3171 
3172 void
3173 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
3174 {
3175         ASSERT(buf->b_hdr != NULL);
3176         ASSERT(buf->b_hdr->b_state != arc_anon);
3177         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
3178         ASSERT(buf->b_efunc == NULL);
3179         ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
3180 
3181         buf->b_efunc = func;
3182         buf->b_private = private;
3183 }
3184 
3185 /*
3186  * Notify the arc that a block was freed, and thus will never be used again.
3187  */
3188 void
3189 arc_freed(spa_t *spa, const blkptr_t *bp)
3190 {
3191         arc_buf_hdr_t *hdr;
3192         kmutex_t *hash_lock;
3193         uint64_t guid = spa_load_guid(spa);
3194 
3195         hdr = buf_hash_find(guid, BP_IDENTITY(bp), BP_PHYSICAL_BIRTH(bp),
3196             &hash_lock);
3197         if (hdr == NULL)
3198                 return;
3199         if (HDR_BUF_AVAILABLE(hdr)) {
3200                 arc_buf_t *buf = hdr->b_buf;
3201                 add_reference(hdr, hash_lock, FTAG);
3202                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3203                 mutex_exit(hash_lock);
3204 
3205                 arc_release(buf, FTAG);
3206                 (void) arc_buf_remove_ref(buf, FTAG);
3207         } else {
3208                 mutex_exit(hash_lock);
3209         }
3210 
3211 }
3212 
3213 /*
3214  * This is used by the DMU to let the ARC know that a buffer is
3215  * being evicted, so the ARC should clean up.  If this arc buf
3216  * is not yet in the evicted state, it will be put there.
3217  */
3218 int
3219 arc_buf_evict(arc_buf_t *buf)
3220 {
3221         arc_buf_hdr_t *hdr;
3222         kmutex_t *hash_lock;
3223         arc_buf_t **bufp;
3224 
3225         mutex_enter(&buf->b_evict_lock);
3226         hdr = buf->b_hdr;
3227         if (hdr == NULL) {
3228                 /*
3229                  * We are in arc_do_user_evicts().
3230                  */
3231                 ASSERT(buf->b_data == NULL);
3232                 mutex_exit(&buf->b_evict_lock);
3233                 return (0);
3234         } else if (buf->b_data == NULL) {
3235                 arc_buf_t copy = *buf; /* structure assignment */
3236                 /*
3237                  * We are on the eviction list; process this buffer now
3238                  * but let arc_do_user_evicts() do the reaping.
3239                  */
3240                 buf->b_efunc = NULL;
3241                 mutex_exit(&buf->b_evict_lock);
3242                 VERIFY(copy.b_efunc(&copy) == 0);
3243                 return (1);
3244         }
3245         hash_lock = HDR_LOCK(hdr);
3246         mutex_enter(hash_lock);
3247         hdr = buf->b_hdr;
3248         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3249 
3250         ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
3251         ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
3252 
3253         /*
3254          * Pull this buffer off of the hdr
3255          */
3256         bufp = &hdr->b_buf;
3257         while (*bufp != buf)
3258                 bufp = &(*bufp)->b_next;
3259         *bufp = buf->b_next;
3260 
3261         ASSERT(buf->b_data != NULL);
3262         arc_buf_destroy(buf, FALSE, FALSE);
3263 
3264         if (hdr->b_datacnt == 0) {
3265                 arc_state_t *old_state = hdr->b_state;
3266                 arc_state_t *evicted_state;
3267 
3268                 ASSERT(hdr->b_buf == NULL);
3269                 ASSERT(refcount_is_zero(&hdr->b_refcnt));
3270 
3271                 evicted_state =
3272                     (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
3273 
3274                 mutex_enter(&old_state->arcs_mtx);
3275                 mutex_enter(&evicted_state->arcs_mtx);
3276 
3277                 arc_change_state(evicted_state, hdr, hash_lock);
3278                 ASSERT(HDR_IN_HASH_TABLE(hdr));
3279                 hdr->b_flags |= ARC_IN_HASH_TABLE;
3280                 hdr->b_flags &= ~ARC_BUF_AVAILABLE;
3281 
3282                 mutex_exit(&evicted_state->arcs_mtx);
3283                 mutex_exit(&old_state->arcs_mtx);
3284         }
3285         mutex_exit(hash_lock);
3286         mutex_exit(&buf->b_evict_lock);
3287 
3288         VERIFY(buf->b_efunc(buf) == 0);
3289         buf->b_efunc = NULL;
3290         buf->b_private = NULL;
3291         buf->b_hdr = NULL;
3292         buf->b_next = NULL;
3293         kmem_cache_free(buf_cache, buf);
3294         return (1);
3295 }
3296 
3297 /*
3298  * Release this buffer from the cache, making it an anonymous buffer.  This
3299  * must be done after a read and prior to modifying the buffer contents.
3300  * If the buffer has more than one reference, we must make
3301  * a new hdr for the buffer.
3302  */
3303 void
3304 arc_release(arc_buf_t *buf, void *tag)
3305 {
3306         arc_buf_hdr_t *hdr;
3307         kmutex_t *hash_lock = NULL;
3308         l2arc_buf_hdr_t *l2hdr;
3309         uint64_t buf_size;
3310 
3311         /*
3312          * It would be nice to assert that if it's DMU metadata (level >
3313          * 0 || it's the dnode file), then it must be syncing context.
3314          * But we don't know that information at this level.
3315          */
3316 
3317         mutex_enter(&buf->b_evict_lock);
3318         hdr = buf->b_hdr;
3319 
3320         /* this buffer is not on any list */
3321         ASSERT(refcount_count(&hdr->b_refcnt) > 0);
3322 
3323         if (hdr->b_state == arc_anon) {
3324                 /* this buffer is already released */
3325                 ASSERT(buf->b_efunc == NULL);
3326         } else {
3327                 hash_lock = HDR_LOCK(hdr);
3328                 mutex_enter(hash_lock);
3329                 hdr = buf->b_hdr;
3330                 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3331         }
3332 
3333         l2hdr = hdr->b_l2hdr;
3334         if (l2hdr) {
3335                 mutex_enter(&l2arc_buflist_mtx);
3336                 hdr->b_l2hdr = NULL;
3337                 list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
3338         }
3339         buf_size = hdr->b_size;
3340 
3341         /*
3342          * Do we have more than one buf?
3343          */
3344         if (hdr->b_datacnt > 1) {
3345                 arc_buf_hdr_t *nhdr;
3346                 arc_buf_t **bufp;
3347                 uint64_t blksz = hdr->b_size;
3348                 uint64_t spa = hdr->b_spa;
3349                 arc_buf_contents_t type = hdr->b_type;
3350                 uint32_t flags = hdr->b_flags;
3351 
3352                 ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
3353                 /*
3354                  * Pull the data off of this hdr and attach it to
3355                  * a new anonymous hdr.
3356                  */
3357                 (void) remove_reference(hdr, hash_lock, tag);
3358                 bufp = &hdr->b_buf;
3359                 while (*bufp != buf)
3360                         bufp = &(*bufp)->b_next;
3361                 *bufp = buf->b_next;
3362                 buf->b_next = NULL;
3363 
3364                 ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
3365                 atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
3366                 if (refcount_is_zero(&hdr->b_refcnt)) {
3367                         uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
3368                         ASSERT3U(*size, >=, hdr->b_size);
3369                         atomic_add_64(size, -hdr->b_size);
3370                 }
3371 
3372                 /*
3373                  * We're releasing a duplicate user data buffer, update
3374                  * our statistics accordingly.
3375                  */
3376                 if (hdr->b_type == ARC_BUFC_DATA) {
3377                         ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
3378                         ARCSTAT_INCR(arcstat_duplicate_buffers_size,
3379                             -hdr->b_size);
3380                 }
3381                 hdr->b_datacnt -= 1;
3382                 arc_cksum_verify(buf);
3383                 arc_buf_unwatch(buf);
3384 
3385                 mutex_exit(hash_lock);
3386 
3387                 nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
3388                 nhdr->b_size = blksz;
3389                 nhdr->b_spa = spa;
3390                 nhdr->b_type = type;
3391                 nhdr->b_buf = buf;
3392                 nhdr->b_state = arc_anon;
3393                 nhdr->b_arc_access = 0;
3394                 nhdr->b_flags = flags & ARC_L2_WRITING;
3395                 nhdr->b_l2hdr = NULL;
3396                 nhdr->b_datacnt = 1;
3397                 nhdr->b_freeze_cksum = NULL;
3398                 (void) refcount_add(&nhdr->b_refcnt, tag);
3399                 buf->b_hdr = nhdr;
3400                 mutex_exit(&buf->b_evict_lock);
3401                 atomic_add_64(&arc_anon->arcs_size, blksz);
3402         } else {
3403                 mutex_exit(&buf->b_evict_lock);
3404                 ASSERT(refcount_count(&hdr->b_refcnt) == 1);
3405                 ASSERT(!list_link_active(&hdr->b_arc_node));
3406                 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3407                 if (hdr->b_state != arc_anon)
3408                         arc_change_state(arc_anon, hdr, hash_lock);
3409                 hdr->b_arc_access = 0;
3410                 if (hash_lock)
3411                         mutex_exit(hash_lock);
3412 
3413                 buf_discard_identity(hdr);
3414                 arc_buf_thaw(buf);
3415         }
3416         buf->b_efunc = NULL;
3417         buf->b_private = NULL;
3418 
3419         if (l2hdr) {
3420                 ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
3421                 kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
3422                 ARCSTAT_INCR(arcstat_l2_size, -buf_size);
3423                 mutex_exit(&l2arc_buflist_mtx);
3424         }
3425 }
3426 
3427 int
3428 arc_released(arc_buf_t *buf)
3429 {
3430         int released;
3431 
3432         mutex_enter(&buf->b_evict_lock);
3433         released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
3434         mutex_exit(&buf->b_evict_lock);
3435         return (released);
3436 }
3437 
3438 int
3439 arc_has_callback(arc_buf_t *buf)
3440 {
3441         int callback;
3442 
3443         mutex_enter(&buf->b_evict_lock);
3444         callback = (buf->b_efunc != NULL);
3445         mutex_exit(&buf->b_evict_lock);
3446         return (callback);
3447 }
3448 
3449 #ifdef ZFS_DEBUG
3450 int
3451 arc_referenced(arc_buf_t *buf)
3452 {
3453         int referenced;
3454 
3455         mutex_enter(&buf->b_evict_lock);
3456         referenced = (refcount_count(&buf->b_hdr->b_refcnt));
3457         mutex_exit(&buf->b_evict_lock);
3458         return (referenced);
3459 }
3460 #endif
3461 
3462 static void
3463 arc_write_ready(zio_t *zio)
3464 {
3465         arc_write_callback_t *callback = zio->io_private;
3466         arc_buf_t *buf = callback->awcb_buf;
3467         arc_buf_hdr_t *hdr = buf->b_hdr;
3468 
3469         ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
3470         callback->awcb_ready(zio, buf, callback->awcb_private);
3471 
3472         /*
3473          * If the IO is already in progress, then this is a re-write
3474          * attempt, so we need to thaw and re-compute the cksum.
3475          * It is the responsibility of the callback to handle the
3476          * accounting for any re-write attempt.
3477          */
3478         if (HDR_IO_IN_PROGRESS(hdr)) {
3479                 mutex_enter(&hdr->b_freeze_lock);
3480                 if (hdr->b_freeze_cksum != NULL) {
3481                         kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
3482                         hdr->b_freeze_cksum = NULL;
3483                 }
3484                 mutex_exit(&hdr->b_freeze_lock);
3485         }
3486         arc_cksum_compute(buf, B_FALSE);
3487         hdr->b_flags |= ARC_IO_IN_PROGRESS;
3488 }
3489 
3490 /*
3491  * The SPA calls this callback for each physical write that happens on behalf
3492  * of a logical write.  See the comment in dbuf_write_physdone() for details.
3493  */
3494 static void
3495 arc_write_physdone(zio_t *zio)
3496 {
3497         arc_write_callback_t *cb = zio->io_private;
3498         if (cb->awcb_physdone != NULL)
3499                 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
3500 }
3501 
3502 static void
3503 arc_write_done(zio_t *zio)
3504 {
3505         arc_write_callback_t *callback = zio->io_private;
3506         arc_buf_t *buf = callback->awcb_buf;
3507         arc_buf_hdr_t *hdr = buf->b_hdr;
3508 
3509         ASSERT(hdr->b_acb == NULL);
3510 
3511         if (zio->io_error == 0) {
3512                 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
3513                 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
3514                 hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
3515         } else {
3516                 ASSERT(BUF_EMPTY(hdr));
3517         }
3518 
3519         /*
3520          * If the block to be written was all-zero, we may have
3521          * compressed it away.  In this case no write was performed
3522          * so there will be no dva/birth/checksum.  The buffer must
3523          * therefore remain anonymous (and uncached).
3524          */
3525         if (!BUF_EMPTY(hdr)) {
3526                 arc_buf_hdr_t *exists;
3527                 kmutex_t *hash_lock;
3528 
3529                 ASSERT(zio->io_error == 0);
3530 
3531                 arc_cksum_verify(buf);
3532 
3533                 exists = buf_hash_insert(hdr, &hash_lock);
3534                 if (exists) {
3535                         /*
3536                          * This can only happen if we overwrite for
3537                          * sync-to-convergence, because we remove
3538                          * buffers from the hash table when we arc_free().
3539                          */
3540                         if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
3541                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3542                                         panic("bad overwrite, hdr=%p exists=%p",
3543                                             (void *)hdr, (void *)exists);
3544                                 ASSERT(refcount_is_zero(&exists->b_refcnt));
3545                                 arc_change_state(arc_anon, exists, hash_lock);
3546                                 mutex_exit(hash_lock);
3547                                 arc_hdr_destroy(exists);
3548                                 exists = buf_hash_insert(hdr, &hash_lock);
3549                                 ASSERT3P(exists, ==, NULL);
3550                         } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
3551                                 /* nopwrite */
3552                                 ASSERT(zio->io_prop.zp_nopwrite);
3553                                 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
3554                                         panic("bad nopwrite, hdr=%p exists=%p",
3555                                             (void *)hdr, (void *)exists);
3556                         } else {
3557                                 /* Dedup */
3558                                 ASSERT(hdr->b_datacnt == 1);
3559                                 ASSERT(hdr->b_state == arc_anon);
3560                                 ASSERT(BP_GET_DEDUP(zio->io_bp));
3561                                 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
3562                         }
3563                 }
3564                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3565                 /* if it's not anon, we are doing a scrub */
3566                 if (!exists && hdr->b_state == arc_anon)
3567                         arc_access(hdr, hash_lock);
3568                 mutex_exit(hash_lock);
3569         } else {
3570                 hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
3571         }
3572 
3573         ASSERT(!refcount_is_zero(&hdr->b_refcnt));
3574         callback->awcb_done(zio, buf, callback->awcb_private);
3575 
3576         kmem_free(callback, sizeof (arc_write_callback_t));
3577 }
3578 
3579 zio_t *
3580 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
3581     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
3582     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
3583     arc_done_func_t *done, void *private, zio_priority_t priority,
3584     int zio_flags, const zbookmark_t *zb)
3585 {
3586         arc_buf_hdr_t *hdr = buf->b_hdr;
3587         arc_write_callback_t *callback;
3588         zio_t *zio;
3589 
3590         ASSERT(ready != NULL);
3591         ASSERT(done != NULL);
3592         ASSERT(!HDR_IO_ERROR(hdr));
3593         ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
3594         ASSERT(hdr->b_acb == NULL);
3595         if (l2arc)
3596                 hdr->b_flags |= ARC_L2CACHE;
3597         if (l2arc_compress)
3598                 hdr->b_flags |= ARC_L2COMPRESS;
3599         callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
3600         callback->awcb_ready = ready;
3601         callback->awcb_physdone = physdone;
3602         callback->awcb_done = done;
3603         callback->awcb_private = private;
3604         callback->awcb_buf = buf;
3605 
3606         zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
3607             arc_write_ready, arc_write_physdone, arc_write_done, callback,
3608             priority, zio_flags, zb);
3609 
3610         return (zio);
3611 }
3612 
3613 static int
3614 arc_memory_throttle(uint64_t reserve, uint64_t txg)
3615 {
3616 #ifdef _KERNEL
3617         uint64_t available_memory = ptob(freemem);
3618         static uint64_t page_load = 0;
3619         static uint64_t last_txg = 0;
3620 
3621 #if defined(__i386)
3622         available_memory =
3623             MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
3624 #endif
3625 
3626         if (freemem > physmem * arc_lotsfree_percent / 100)
3627                 return (0);
3628 
3629         if (txg > last_txg) {
3630                 last_txg = txg;
3631                 page_load = 0;
3632         }
3633         /*
3634          * If we are in pageout, we know that memory is already tight,
3635          * the arc is already going to be evicting, so we just want to
3636          * continue to let page writes occur as quickly as possible.
3637          */
3638         if (curproc == proc_pageout) {
3639                 if (page_load > MAX(ptob(minfree), available_memory) / 4)
3640                         return (SET_ERROR(ERESTART));
3641                 /* Note: reserve is inflated, so we deflate */
3642                 page_load += reserve / 8;
3643                 return (0);
3644         } else if (page_load > 0 && arc_reclaim_needed()) {
3645                 /* memory is low, delay before restarting */
3646                 ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
3647                 return (SET_ERROR(EAGAIN));
3648         }
3649         page_load = 0;
3650 #endif
3651         return (0);
3652 }
3653 
3654 void
3655 arc_tempreserve_clear(uint64_t reserve)
3656 {
3657         atomic_add_64(&arc_tempreserve, -reserve);
3658         ASSERT((int64_t)arc_tempreserve >= 0);
3659 }
3660 
3661 int
3662 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
3663 {
3664         int error;
3665         uint64_t anon_size;
3666 
3667         if (reserve > arc_c/4 && !arc_no_grow)
3668                 arc_c = MIN(arc_c_max, reserve * 4);
3669         if (reserve > arc_c)
3670                 return (SET_ERROR(ENOMEM));
3671 
3672         /*
3673          * Don't count loaned bufs as in flight dirty data to prevent long
3674          * network delays from blocking transactions that are ready to be
3675          * assigned to a txg.
3676          */
3677         anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
3678 
3679         /*
3680          * Writes will, almost always, require additional memory allocations
3681          * in order to compress/encrypt/etc the data.  We therefore need to
3682          * make sure that there is sufficient available memory for this.
3683          */
3684         error = arc_memory_throttle(reserve, txg);
3685         if (error != 0)
3686                 return (error);
3687 
3688         /*
3689          * Throttle writes when the amount of dirty data in the cache
3690          * gets too large.  We try to keep the cache less than half full
3691          * of dirty blocks so that our sync times don't grow too large.
3692          * Note: if two requests come in concurrently, we might let them
3693          * both succeed, when one of them should fail.  Not a huge deal.
3694          */
3695 
3696         if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
3697             anon_size > arc_c / 4) {
3698                 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
3699                     "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
3700                     arc_tempreserve>>10,
3701                     arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
3702                     arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
3703                     reserve>>10, arc_c>>10);
3704                 return (SET_ERROR(ERESTART));
3705         }
3706         atomic_add_64(&arc_tempreserve, reserve);
3707         return (0);
3708 }
3709 
3710 void
3711 arc_init(void)
3712 {
3713         mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
3714         cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
3715 
3716         /* Convert seconds to clock ticks */
3717         arc_min_prefetch_lifespan = 1 * hz;
3718 
3719         /* Start out with 1/8 of all memory */
3720         arc_c = physmem * PAGESIZE / 8;
3721 
3722 #ifdef _KERNEL
3723         /*
3724          * On architectures where the physical memory can be larger
3725          * than the addressable space (intel in 32-bit mode), we may
3726          * need to limit the cache to 1/8 of VM size.
3727          */
3728         arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
3729 #endif
3730 
3731         /* set min cache to 1/32 of all memory, or 64MB, whichever is more */
3732         arc_c_min = MAX(arc_c / 4, 64<<20);
3733         /* set max to 3/4 of all memory, or all but 1GB, whichever is more */
3734         if (arc_c * 8 >= 1<<30)
3735                 arc_c_max = (arc_c * 8) - (1<<30);
3736         else
3737                 arc_c_max = arc_c_min;
3738         arc_c_max = MAX(arc_c * 6, arc_c_max);
3739 
3740         /*
3741          * Allow the tunables to override our calculations if they are
3742          * reasonable (ie. over 64MB)
3743          */
3744         if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
3745                 arc_c_max = zfs_arc_max;
3746         if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
3747                 arc_c_min = zfs_arc_min;
3748 
3749         arc_c = arc_c_max;
3750         arc_p = (arc_c >> 1);
3751 
3752         /* limit meta-data to 1/4 of the arc capacity */
3753         arc_meta_limit = arc_c_max / 4;
3754 
3755         /* Allow the tunable to override if it is reasonable */
3756         if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
3757                 arc_meta_limit = zfs_arc_meta_limit;
3758 
3759         if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
3760                 arc_c_min = arc_meta_limit / 2;
3761 
3762         if (zfs_arc_grow_retry > 0)
3763                 arc_grow_retry = zfs_arc_grow_retry;
3764 
3765         if (zfs_arc_shrink_shift > 0)
3766                 arc_shrink_shift = zfs_arc_shrink_shift;
3767 
3768         if (zfs_arc_p_min_shift > 0)
3769                 arc_p_min_shift = zfs_arc_p_min_shift;
3770 
3771         /* if kmem_flags are set, lets try to use less memory */
3772         if (kmem_debugging())
3773                 arc_c = arc_c / 2;
3774         if (arc_c < arc_c_min)
3775                 arc_c = arc_c_min;
3776 
3777         arc_anon = &ARC_anon;
3778         arc_mru = &ARC_mru;
3779         arc_mru_ghost = &ARC_mru_ghost;
3780         arc_mfu = &ARC_mfu;
3781         arc_mfu_ghost = &ARC_mfu_ghost;
3782         arc_l2c_only = &ARC_l2c_only;
3783         arc_size = 0;
3784 
3785         mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3786         mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3787         mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3788         mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3789         mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3790         mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
3791 
3792         list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
3793             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3794         list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
3795             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3796         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
3797             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3798         list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
3799             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3800         list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
3801             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3802         list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
3803             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3804         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
3805             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3806         list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
3807             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3808         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
3809             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3810         list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
3811             sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
3812 
3813         buf_init();
3814 
3815         arc_thread_exit = 0;
3816         arc_eviction_list = NULL;
3817         mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
3818         bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
3819 
3820         arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
3821             sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
3822 
3823         if (arc_ksp != NULL) {
3824                 arc_ksp->ks_data = &arc_stats;
3825                 kstat_install(arc_ksp);
3826         }
3827 
3828         (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
3829             TS_RUN, minclsyspri);
3830 
3831         arc_dead = FALSE;
3832         arc_warm = B_FALSE;
3833 
3834         /*
3835          * Calculate maximum amount of dirty data per pool.
3836          *
3837          * If it has been set by /etc/system, take that.
3838          * Otherwise, use a percentage of physical memory defined by
3839          * zfs_dirty_data_max_percent (default 10%) with a cap at
3840          * zfs_dirty_data_max_max (default 4GB).
3841          */
3842         if (zfs_dirty_data_max == 0) {
3843                 zfs_dirty_data_max = physmem * PAGESIZE *
3844                     zfs_dirty_data_max_percent / 100;
3845                 zfs_dirty_data_max = MIN(zfs_dirty_data_max,
3846                     zfs_dirty_data_max_max);
3847         }
3848 }
3849 
3850 void
3851 arc_fini(void)
3852 {
3853         mutex_enter(&arc_reclaim_thr_lock);
3854         arc_thread_exit = 1;
3855         while (arc_thread_exit != 0)
3856                 cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
3857         mutex_exit(&arc_reclaim_thr_lock);
3858 
3859         arc_flush(NULL);
3860 
3861         arc_dead = TRUE;
3862 
3863         if (arc_ksp != NULL) {
3864                 kstat_delete(arc_ksp);
3865                 arc_ksp = NULL;
3866         }
3867 
3868         mutex_destroy(&arc_eviction_mtx);
3869         mutex_destroy(&arc_reclaim_thr_lock);
3870         cv_destroy(&arc_reclaim_thr_cv);
3871 
3872         list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
3873         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
3874         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
3875         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
3876         list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
3877         list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
3878         list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
3879         list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
3880 
3881         mutex_destroy(&arc_anon->arcs_mtx);
3882         mutex_destroy(&arc_mru->arcs_mtx);
3883         mutex_destroy(&arc_mru_ghost->arcs_mtx);
3884         mutex_destroy(&arc_mfu->arcs_mtx);
3885         mutex_destroy(&arc_mfu_ghost->arcs_mtx);
3886         mutex_destroy(&arc_l2c_only->arcs_mtx);
3887 
3888         buf_fini();
3889 
3890         ASSERT(arc_loaned_bytes == 0);
3891 }
3892 
3893 /*
3894  * Level 2 ARC
3895  *
3896  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
3897  * It uses dedicated storage devices to hold cached data, which are populated
3898  * using large infrequent writes.  The main role of this cache is to boost
3899  * the performance of random read workloads.  The intended L2ARC devices
3900  * include short-stroked disks, solid state disks, and other media with
3901  * substantially faster read latency than disk.
3902  *
3903  *                 +-----------------------+
3904  *                 |         ARC           |
3905  *                 +-----------------------+
3906  *                    |         ^     ^
3907  *                    |         |     |
3908  *      l2arc_feed_thread()    arc_read()
3909  *                    |         |     |
3910  *                    |  l2arc read   |
3911  *                    V         |     |
3912  *               +---------------+    |
3913  *               |     L2ARC     |    |
3914  *               +---------------+    |
3915  *                   |    ^           |
3916  *          l2arc_write() |           |
3917  *                   |    |           |
3918  *                   V    |           |
3919  *                 +-------+      +-------+
3920  *                 | vdev  |      | vdev  |
3921  *                 | cache |      | cache |
3922  *                 +-------+      +-------+
3923  *                 +=========+     .-----.
3924  *                 :  L2ARC  :    |-_____-|
3925  *                 : devices :    | Disks |
3926  *                 +=========+    `-_____-'
3927  *
3928  * Read requests are satisfied from the following sources, in order:
3929  *
3930  *      1) ARC
3931  *      2) vdev cache of L2ARC devices
3932  *      3) L2ARC devices
3933  *      4) vdev cache of disks
3934  *      5) disks
3935  *
3936  * Some L2ARC device types exhibit extremely slow write performance.
3937  * To accommodate for this there are some significant differences between
3938  * the L2ARC and traditional cache design:
3939  *
3940  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
3941  * the ARC behave as usual, freeing buffers and placing headers on ghost
3942  * lists.  The ARC does not send buffers to the L2ARC during eviction as
3943  * this would add inflated write latencies for all ARC memory pressure.
3944  *
3945  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
3946  * It does this by periodically scanning buffers from the eviction-end of
3947  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
3948  * not already there. It scans until a headroom of buffers is satisfied,
3949  * which itself is a buffer for ARC eviction. If a compressible buffer is
3950  * found during scanning and selected for writing to an L2ARC device, we
3951  * temporarily boost scanning headroom during the next scan cycle to make
3952  * sure we adapt to compression effects (which might significantly reduce
3953  * the data volume we write to L2ARC). The thread that does this is
3954  * l2arc_feed_thread(), illustrated below; example sizes are included to
3955  * provide a better sense of ratio than this diagram:
3956  *
3957  *             head -->                        tail
3958  *              +---------------------+----------+
3959  *      ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
3960  *              +---------------------+----------+   |   o L2ARC eligible
3961  *      ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
3962  *              +---------------------+----------+   |
3963  *                   15.9 Gbytes      ^ 32 Mbytes    |
3964  *                                 headroom          |
3965  *                                            l2arc_feed_thread()
3966  *                                                   |
3967  *                       l2arc write hand <--[oooo]--'
3968  *                               |           8 Mbyte
3969  *                               |          write max
3970  *                               V
3971  *                +==============================+
3972  *      L2ARC dev |####|#|###|###|    |####| ... |
3973  *                +==============================+
3974  *                           32 Gbytes
3975  *
3976  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
3977  * evicted, then the L2ARC has cached a buffer much sooner than it probably
3978  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
3979  * safe to say that this is an uncommon case, since buffers at the end of
3980  * the ARC lists have moved there due to inactivity.
3981  *
3982  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
3983  * then the L2ARC simply misses copying some buffers.  This serves as a
3984  * pressure valve to prevent heavy read workloads from both stalling the ARC
3985  * with waits and clogging the L2ARC with writes.  This also helps prevent
3986  * the potential for the L2ARC to churn if it attempts to cache content too
3987  * quickly, such as during backups of the entire pool.
3988  *
3989  * 5. After system boot and before the ARC has filled main memory, there are
3990  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
3991  * lists can remain mostly static.  Instead of searching from tail of these
3992  * lists as pictured, the l2arc_feed_thread() will search from the list heads
3993  * for eligible buffers, greatly increasing its chance of finding them.
3994  *
3995  * The L2ARC device write speed is also boosted during this time so that
3996  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
3997  * there are no L2ARC reads, and no fear of degrading read performance
3998  * through increased writes.
3999  *
4000  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
4001  * the vdev queue can aggregate them into larger and fewer writes.  Each
4002  * device is written to in a rotor fashion, sweeping writes through
4003  * available space then repeating.
4004  *
4005  * 7. The L2ARC does not store dirty content.  It never needs to flush
4006  * write buffers back to disk based storage.
4007  *
4008  * 8. If an ARC buffer is written (and dirtied) which also exists in the
4009  * L2ARC, the now stale L2ARC buffer is immediately dropped.
4010  *
4011  * The performance of the L2ARC can be tweaked by a number of tunables, which
4012  * may be necessary for different workloads:
4013  *
4014  *      l2arc_write_max         max write bytes per interval
4015  *      l2arc_write_boost       extra write bytes during device warmup
4016  *      l2arc_noprefetch        skip caching prefetched buffers
4017  *      l2arc_headroom          number of max device writes to precache
4018  *      l2arc_headroom_boost    when we find compressed buffers during ARC
4019  *                              scanning, we multiply headroom by this
4020  *                              percentage factor for the next scan cycle,
4021  *                              since more compressed buffers are likely to
4022  *                              be present
4023  *      l2arc_feed_secs         seconds between L2ARC writing
4024  *
4025  * Tunables may be removed or added as future performance improvements are
4026  * integrated, and also may become zpool properties.
4027  *
4028  * There are three key functions that control how the L2ARC warms up:
4029  *
4030  *      l2arc_write_eligible()  check if a buffer is eligible to cache
4031  *      l2arc_write_size()      calculate how much to write
4032  *      l2arc_write_interval()  calculate sleep delay between writes
4033  *
4034  * These three functions determine what to write, how much, and how quickly
4035  * to send writes.
4036  */
4037 
4038 static boolean_t
4039 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *ab)
4040 {
4041         /*
4042          * A buffer is *not* eligible for the L2ARC if it:
4043          * 1. belongs to a different spa.
4044          * 2. is already cached on the L2ARC.
4045          * 3. has an I/O in progress (it may be an incomplete read).
4046          * 4. is flagged not eligible (zfs property).
4047          */
4048         if (ab->b_spa != spa_guid || ab->b_l2hdr != NULL ||
4049             HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab))
4050                 return (B_FALSE);
4051 
4052         return (B_TRUE);
4053 }
4054 
4055 static uint64_t
4056 l2arc_write_size(void)
4057 {
4058         uint64_t size;
4059 
4060         /*
4061          * Make sure our globals have meaningful values in case the user
4062          * altered them.
4063          */
4064         size = l2arc_write_max;
4065         if (size == 0) {
4066                 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
4067                     "be greater than zero, resetting it to the default (%d)",
4068                     L2ARC_WRITE_SIZE);
4069                 size = l2arc_write_max = L2ARC_WRITE_SIZE;
4070         }
4071 
4072         if (arc_warm == B_FALSE)
4073                 size += l2arc_write_boost;
4074 
4075         return (size);
4076 
4077 }
4078 
4079 static clock_t
4080 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
4081 {
4082         clock_t interval, next, now;
4083 
4084         /*
4085          * If the ARC lists are busy, increase our write rate; if the
4086          * lists are stale, idle back.  This is achieved by checking
4087          * how much we previously wrote - if it was more than half of
4088          * what we wanted, schedule the next write much sooner.
4089          */
4090         if (l2arc_feed_again && wrote > (wanted / 2))
4091                 interval = (hz * l2arc_feed_min_ms) / 1000;
4092         else
4093                 interval = hz * l2arc_feed_secs;
4094 
4095         now = ddi_get_lbolt();
4096         next = MAX(now, MIN(now + interval, began + interval));
4097 
4098         return (next);
4099 }
4100 
4101 static void
4102 l2arc_hdr_stat_add(void)
4103 {
4104         ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
4105         ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
4106 }
4107 
4108 static void
4109 l2arc_hdr_stat_remove(void)
4110 {
4111         ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
4112         ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
4113 }
4114 
4115 /*
4116  * Cycle through L2ARC devices.  This is how L2ARC load balances.
4117  * If a device is returned, this also returns holding the spa config lock.
4118  */
4119 static l2arc_dev_t *
4120 l2arc_dev_get_next(void)
4121 {
4122         l2arc_dev_t *first, *next = NULL;
4123 
4124         /*
4125          * Lock out the removal of spas (spa_namespace_lock), then removal
4126          * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
4127          * both locks will be dropped and a spa config lock held instead.
4128          */
4129         mutex_enter(&spa_namespace_lock);
4130         mutex_enter(&l2arc_dev_mtx);
4131 
4132         /* if there are no vdevs, there is nothing to do */
4133         if (l2arc_ndev == 0)
4134                 goto out;
4135 
4136         first = NULL;
4137         next = l2arc_dev_last;
4138         do {
4139                 /* loop around the list looking for a non-faulted vdev */
4140                 if (next == NULL) {
4141                         next = list_head(l2arc_dev_list);
4142                 } else {
4143                         next = list_next(l2arc_dev_list, next);
4144                         if (next == NULL)
4145                                 next = list_head(l2arc_dev_list);
4146                 }
4147 
4148                 /* if we have come back to the start, bail out */
4149                 if (first == NULL)
4150                         first = next;
4151                 else if (next == first)
4152                         break;
4153 
4154         } while (vdev_is_dead(next->l2ad_vdev));
4155 
4156         /* if we were unable to find any usable vdevs, return NULL */
4157         if (vdev_is_dead(next->l2ad_vdev))
4158                 next = NULL;
4159 
4160         l2arc_dev_last = next;
4161 
4162 out:
4163         mutex_exit(&l2arc_dev_mtx);
4164 
4165         /*
4166          * Grab the config lock to prevent the 'next' device from being
4167          * removed while we are writing to it.
4168          */
4169         if (next != NULL)
4170                 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
4171         mutex_exit(&spa_namespace_lock);
4172 
4173         return (next);
4174 }
4175 
4176 /*
4177  * Free buffers that were tagged for destruction.
4178  */
4179 static void
4180 l2arc_do_free_on_write()
4181 {
4182         list_t *buflist;
4183         l2arc_data_free_t *df, *df_prev;
4184 
4185         mutex_enter(&l2arc_free_on_write_mtx);
4186         buflist = l2arc_free_on_write;
4187 
4188         for (df = list_tail(buflist); df; df = df_prev) {
4189                 df_prev = list_prev(buflist, df);
4190                 ASSERT(df->l2df_data != NULL);
4191                 ASSERT(df->l2df_func != NULL);
4192                 df->l2df_func(df->l2df_data, df->l2df_size);
4193                 list_remove(buflist, df);
4194                 kmem_free(df, sizeof (l2arc_data_free_t));
4195         }
4196 
4197         mutex_exit(&l2arc_free_on_write_mtx);
4198 }
4199 
4200 /*
4201  * A write to a cache device has completed.  Update all headers to allow
4202  * reads from these buffers to begin.
4203  */
4204 static void
4205 l2arc_write_done(zio_t *zio)
4206 {
4207         l2arc_write_callback_t *cb;
4208         l2arc_dev_t *dev;
4209         list_t *buflist;
4210         arc_buf_hdr_t *head, *ab, *ab_prev;
4211         l2arc_buf_hdr_t *abl2;
4212         kmutex_t *hash_lock;
4213 
4214         cb = zio->io_private;
4215         ASSERT(cb != NULL);
4216         dev = cb->l2wcb_dev;
4217         ASSERT(dev != NULL);
4218         head = cb->l2wcb_head;
4219         ASSERT(head != NULL);
4220         buflist = dev->l2ad_buflist;
4221         ASSERT(buflist != NULL);
4222         DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
4223             l2arc_write_callback_t *, cb);
4224 
4225         if (zio->io_error != 0)
4226                 ARCSTAT_BUMP(arcstat_l2_writes_error);
4227 
4228         mutex_enter(&l2arc_buflist_mtx);
4229 
4230         /*
4231          * All writes completed, or an error was hit.
4232          */
4233         for (ab = list_prev(buflist, head); ab; ab = ab_prev) {
4234                 ab_prev = list_prev(buflist, ab);
4235 
4236                 hash_lock = HDR_LOCK(ab);
4237                 if (!mutex_tryenter(hash_lock)) {
4238                         /*
4239                          * This buffer misses out.  It may be in a stage
4240                          * of eviction.  Its ARC_L2_WRITING flag will be
4241                          * left set, denying reads to this buffer.
4242                          */
4243                         ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
4244                         continue;
4245                 }
4246 
4247                 abl2 = ab->b_l2hdr;
4248 
4249                 /*
4250                  * Release the temporary compressed buffer as soon as possible.
4251                  */
4252                 if (abl2->b_compress != ZIO_COMPRESS_OFF)
4253                         l2arc_release_cdata_buf(ab);
4254 
4255                 if (zio->io_error != 0) {
4256                         /*
4257                          * Error - drop L2ARC entry.
4258                          */
4259                         list_remove(buflist, ab);
4260                         ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4261                         ab->b_l2hdr = NULL;
4262                         kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4263                         ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4264                 }
4265 
4266                 /*
4267                  * Allow ARC to begin reads to this L2ARC entry.
4268                  */
4269                 ab->b_flags &= ~ARC_L2_WRITING;
4270 
4271                 mutex_exit(hash_lock);
4272         }
4273 
4274         atomic_inc_64(&l2arc_writes_done);
4275         list_remove(buflist, head);
4276         kmem_cache_free(hdr_cache, head);
4277         mutex_exit(&l2arc_buflist_mtx);
4278 
4279         l2arc_do_free_on_write();
4280 
4281         kmem_free(cb, sizeof (l2arc_write_callback_t));
4282 }
4283 
4284 /*
4285  * A read to a cache device completed.  Validate buffer contents before
4286  * handing over to the regular ARC routines.
4287  */
4288 static void
4289 l2arc_read_done(zio_t *zio)
4290 {
4291         l2arc_read_callback_t *cb;
4292         arc_buf_hdr_t *hdr;
4293         arc_buf_t *buf;
4294         kmutex_t *hash_lock;
4295         int equal;
4296 
4297         ASSERT(zio->io_vd != NULL);
4298         ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
4299 
4300         spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
4301 
4302         cb = zio->io_private;
4303         ASSERT(cb != NULL);
4304         buf = cb->l2rcb_buf;
4305         ASSERT(buf != NULL);
4306 
4307         hash_lock = HDR_LOCK(buf->b_hdr);
4308         mutex_enter(hash_lock);
4309         hdr = buf->b_hdr;
4310         ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4311 
4312         /*
4313          * If the buffer was compressed, decompress it first.
4314          */
4315         if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
4316                 l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
4317         ASSERT(zio->io_data != NULL);
4318 
4319         /*
4320          * Check this survived the L2ARC journey.
4321          */
4322         equal = arc_cksum_equal(buf);
4323         if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
4324                 mutex_exit(hash_lock);
4325                 zio->io_private = buf;
4326                 zio->io_bp_copy = cb->l2rcb_bp;   /* XXX fix in L2ARC 2.0 */
4327                 zio->io_bp = &zio->io_bp_copy;        /* XXX fix in L2ARC 2.0 */
4328                 arc_read_done(zio);
4329         } else {
4330                 mutex_exit(hash_lock);
4331                 /*
4332                  * Buffer didn't survive caching.  Increment stats and
4333                  * reissue to the original storage device.
4334                  */
4335                 if (zio->io_error != 0) {
4336                         ARCSTAT_BUMP(arcstat_l2_io_error);
4337                 } else {
4338                         zio->io_error = SET_ERROR(EIO);
4339                 }
4340                 if (!equal)
4341                         ARCSTAT_BUMP(arcstat_l2_cksum_bad);
4342 
4343                 /*
4344                  * If there's no waiter, issue an async i/o to the primary
4345                  * storage now.  If there *is* a waiter, the caller must
4346                  * issue the i/o in a context where it's OK to block.
4347                  */
4348                 if (zio->io_waiter == NULL) {
4349                         zio_t *pio = zio_unique_parent(zio);
4350 
4351                         ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
4352 
4353                         zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
4354                             buf->b_data, zio->io_size, arc_read_done, buf,
4355                             zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
4356                 }
4357         }
4358 
4359         kmem_free(cb, sizeof (l2arc_read_callback_t));
4360 }
4361 
4362 /*
4363  * This is the list priority from which the L2ARC will search for pages to
4364  * cache.  This is used within loops (0..3) to cycle through lists in the
4365  * desired order.  This order can have a significant effect on cache
4366  * performance.
4367  *
4368  * Currently the metadata lists are hit first, MFU then MRU, followed by
4369  * the data lists.  This function returns a locked list, and also returns
4370  * the lock pointer.
4371  */
4372 static list_t *
4373 l2arc_list_locked(int list_num, kmutex_t **lock)
4374 {
4375         list_t *list = NULL;
4376 
4377         ASSERT(list_num >= 0 && list_num <= 3);
4378 
4379         switch (list_num) {
4380         case 0:
4381                 list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
4382                 *lock = &arc_mfu->arcs_mtx;
4383                 break;
4384         case 1:
4385                 list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
4386                 *lock = &arc_mru->arcs_mtx;
4387                 break;
4388         case 2:
4389                 list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
4390                 *lock = &arc_mfu->arcs_mtx;
4391                 break;
4392         case 3:
4393                 list = &arc_mru->arcs_list[ARC_BUFC_DATA];
4394                 *lock = &arc_mru->arcs_mtx;
4395                 break;
4396         }
4397 
4398         ASSERT(!(MUTEX_HELD(*lock)));
4399         mutex_enter(*lock);
4400         return (list);
4401 }
4402 
4403 /*
4404  * Evict buffers from the device write hand to the distance specified in
4405  * bytes.  This distance may span populated buffers, it may span nothing.
4406  * This is clearing a region on the L2ARC device ready for writing.
4407  * If the 'all' boolean is set, every buffer is evicted.
4408  */
4409 static void
4410 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
4411 {
4412         list_t *buflist;
4413         l2arc_buf_hdr_t *abl2;
4414         arc_buf_hdr_t *ab, *ab_prev;
4415         kmutex_t *hash_lock;
4416         uint64_t taddr;
4417 
4418         buflist = dev->l2ad_buflist;
4419 
4420         if (buflist == NULL)
4421                 return;
4422 
4423         if (!all && dev->l2ad_first) {
4424                 /*
4425                  * This is the first sweep through the device.  There is
4426                  * nothing to evict.
4427                  */
4428                 return;
4429         }
4430 
4431         if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
4432                 /*
4433                  * When nearing the end of the device, evict to the end
4434                  * before the device write hand jumps to the start.
4435                  */
4436                 taddr = dev->l2ad_end;
4437         } else {
4438                 taddr = dev->l2ad_hand + distance;
4439         }
4440         DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
4441             uint64_t, taddr, boolean_t, all);
4442 
4443 top:
4444         mutex_enter(&l2arc_buflist_mtx);
4445         for (ab = list_tail(buflist); ab; ab = ab_prev) {
4446                 ab_prev = list_prev(buflist, ab);
4447 
4448                 hash_lock = HDR_LOCK(ab);
4449                 if (!mutex_tryenter(hash_lock)) {
4450                         /*
4451                          * Missed the hash lock.  Retry.
4452                          */
4453                         ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
4454                         mutex_exit(&l2arc_buflist_mtx);
4455                         mutex_enter(hash_lock);
4456                         mutex_exit(hash_lock);
4457                         goto top;
4458                 }
4459 
4460                 if (HDR_L2_WRITE_HEAD(ab)) {
4461                         /*
4462                          * We hit a write head node.  Leave it for
4463                          * l2arc_write_done().
4464                          */
4465                         list_remove(buflist, ab);
4466                         mutex_exit(hash_lock);
4467                         continue;
4468                 }
4469 
4470                 if (!all && ab->b_l2hdr != NULL &&
4471                     (ab->b_l2hdr->b_daddr > taddr ||
4472                     ab->b_l2hdr->b_daddr < dev->l2ad_hand)) {
4473                         /*
4474                          * We've evicted to the target address,
4475                          * or the end of the device.
4476                          */
4477                         mutex_exit(hash_lock);
4478                         break;
4479                 }
4480 
4481                 if (HDR_FREE_IN_PROGRESS(ab)) {
4482                         /*
4483                          * Already on the path to destruction.
4484                          */
4485                         mutex_exit(hash_lock);
4486                         continue;
4487                 }
4488 
4489                 if (ab->b_state == arc_l2c_only) {
4490                         ASSERT(!HDR_L2_READING(ab));
4491                         /*
4492                          * This doesn't exist in the ARC.  Destroy.
4493                          * arc_hdr_destroy() will call list_remove()
4494                          * and decrement arcstat_l2_size.
4495                          */
4496                         arc_change_state(arc_anon, ab, hash_lock);
4497                         arc_hdr_destroy(ab);
4498                 } else {
4499                         /*
4500                          * Invalidate issued or about to be issued
4501                          * reads, since we may be about to write
4502                          * over this location.
4503                          */
4504                         if (HDR_L2_READING(ab)) {
4505                                 ARCSTAT_BUMP(arcstat_l2_evict_reading);
4506                                 ab->b_flags |= ARC_L2_EVICTED;
4507                         }
4508 
4509                         /*
4510                          * Tell ARC this no longer exists in L2ARC.
4511                          */
4512                         if (ab->b_l2hdr != NULL) {
4513                                 abl2 = ab->b_l2hdr;
4514                                 ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
4515                                 ab->b_l2hdr = NULL;
4516                                 kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
4517                                 ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
4518                         }
4519                         list_remove(buflist, ab);
4520 
4521                         /*
4522                          * This may have been leftover after a
4523                          * failed write.
4524                          */
4525                         ab->b_flags &= ~ARC_L2_WRITING;
4526                 }
4527                 mutex_exit(hash_lock);
4528         }
4529         mutex_exit(&l2arc_buflist_mtx);
4530 
4531         vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
4532         dev->l2ad_evict = taddr;
4533 }
4534 
4535 /*
4536  * Find and write ARC buffers to the L2ARC device.
4537  *
4538  * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
4539  * for reading until they have completed writing.
4540  * The headroom_boost is an in-out parameter used to maintain headroom boost
4541  * state between calls to this function.
4542  *
4543  * Returns the number of bytes actually written (which may be smaller than
4544  * the delta by which the device hand has changed due to alignment).
4545  */
4546 static uint64_t
4547 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
4548     boolean_t *headroom_boost)
4549 {
4550         arc_buf_hdr_t *ab, *ab_prev, *head;
4551         list_t *list;
4552         uint64_t write_asize, write_psize, write_sz, headroom,
4553             buf_compress_minsz;
4554         void *buf_data;
4555         kmutex_t *list_lock;
4556         boolean_t full;
4557         l2arc_write_callback_t *cb;
4558         zio_t *pio, *wzio;
4559         uint64_t guid = spa_load_guid(spa);
4560         const boolean_t do_headroom_boost = *headroom_boost;
4561 
4562         ASSERT(dev->l2ad_vdev != NULL);
4563 
4564         /* Lower the flag now, we might want to raise it again later. */
4565         *headroom_boost = B_FALSE;
4566 
4567         pio = NULL;
4568         write_sz = write_asize = write_psize = 0;
4569         full = B_FALSE;
4570         head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
4571         head->b_flags |= ARC_L2_WRITE_HEAD;
4572 
4573         /*
4574          * We will want to try to compress buffers that are at least 2x the
4575          * device sector size.
4576          */
4577         buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
4578 
4579         /*
4580          * Copy buffers for L2ARC writing.
4581          */
4582         mutex_enter(&l2arc_buflist_mtx);
4583         for (int try = 0; try <= 3; try++) {
4584                 uint64_t passed_sz = 0;
4585 
4586                 list = l2arc_list_locked(try, &list_lock);
4587 
4588                 /*
4589                  * L2ARC fast warmup.
4590                  *
4591                  * Until the ARC is warm and starts to evict, read from the
4592                  * head of the ARC lists rather than the tail.
4593                  */
4594                 if (arc_warm == B_FALSE)
4595                         ab = list_head(list);
4596                 else
4597                         ab = list_tail(list);
4598 
4599                 headroom = target_sz * l2arc_headroom;
4600                 if (do_headroom_boost)
4601                         headroom = (headroom * l2arc_headroom_boost) / 100;
4602 
4603                 for (; ab; ab = ab_prev) {
4604                         l2arc_buf_hdr_t *l2hdr;
4605                         kmutex_t *hash_lock;
4606                         uint64_t buf_sz;
4607 
4608                         if (arc_warm == B_FALSE)
4609                                 ab_prev = list_next(list, ab);
4610                         else
4611                                 ab_prev = list_prev(list, ab);
4612 
4613                         hash_lock = HDR_LOCK(ab);
4614                         if (!mutex_tryenter(hash_lock)) {
4615                                 /*
4616                                  * Skip this buffer rather than waiting.
4617                                  */
4618                                 continue;
4619                         }
4620 
4621                         passed_sz += ab->b_size;
4622                         if (passed_sz > headroom) {
4623                                 /*
4624                                  * Searched too far.
4625                                  */
4626                                 mutex_exit(hash_lock);
4627                                 break;
4628                         }
4629 
4630                         if (!l2arc_write_eligible(guid, ab)) {
4631                                 mutex_exit(hash_lock);
4632                                 continue;
4633                         }
4634 
4635                         if ((write_sz + ab->b_size) > target_sz) {
4636                                 full = B_TRUE;
4637                                 mutex_exit(hash_lock);
4638                                 break;
4639                         }
4640 
4641                         if (pio == NULL) {
4642                                 /*
4643                                  * Insert a dummy header on the buflist so
4644                                  * l2arc_write_done() can find where the
4645                                  * write buffers begin without searching.
4646                                  */
4647                                 list_insert_head(dev->l2ad_buflist, head);
4648 
4649                                 cb = kmem_alloc(
4650                                     sizeof (l2arc_write_callback_t), KM_SLEEP);
4651                                 cb->l2wcb_dev = dev;
4652                                 cb->l2wcb_head = head;
4653                                 pio = zio_root(spa, l2arc_write_done, cb,
4654                                     ZIO_FLAG_CANFAIL);
4655                         }
4656 
4657                         /*
4658                          * Create and add a new L2ARC header.
4659                          */
4660                         l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
4661                         l2hdr->b_dev = dev;
4662                         ab->b_flags |= ARC_L2_WRITING;
4663 
4664                         /*
4665                          * Temporarily stash the data buffer in b_tmp_cdata.
4666                          * The subsequent write step will pick it up from
4667                          * there. This is because can't access ab->b_buf
4668                          * without holding the hash_lock, which we in turn
4669                          * can't access without holding the ARC list locks
4670                          * (which we want to avoid during compression/writing).
4671                          */
4672                         l2hdr->b_compress = ZIO_COMPRESS_OFF;
4673                         l2hdr->b_asize = ab->b_size;
4674                         l2hdr->b_tmp_cdata = ab->b_buf->b_data;
4675 
4676                         buf_sz = ab->b_size;
4677                         ab->b_l2hdr = l2hdr;
4678 
4679                         list_insert_head(dev->l2ad_buflist, ab);
4680 
4681                         /*
4682                          * Compute and store the buffer cksum before
4683                          * writing.  On debug the cksum is verified first.
4684                          */
4685                         arc_cksum_verify(ab->b_buf);
4686                         arc_cksum_compute(ab->b_buf, B_TRUE);
4687 
4688                         mutex_exit(hash_lock);
4689 
4690                         write_sz += buf_sz;
4691                 }
4692 
4693                 mutex_exit(list_lock);
4694 
4695                 if (full == B_TRUE)
4696                         break;
4697         }
4698 
4699         /* No buffers selected for writing? */
4700         if (pio == NULL) {
4701                 ASSERT0(write_sz);
4702                 mutex_exit(&l2arc_buflist_mtx);
4703                 kmem_cache_free(hdr_cache, head);
4704                 return (0);
4705         }
4706 
4707         /*
4708          * Now start writing the buffers. We're starting at the write head
4709          * and work backwards, retracing the course of the buffer selector
4710          * loop above.
4711          */
4712         for (ab = list_prev(dev->l2ad_buflist, head); ab;
4713             ab = list_prev(dev->l2ad_buflist, ab)) {
4714                 l2arc_buf_hdr_t *l2hdr;
4715                 uint64_t buf_sz;
4716 
4717                 /*
4718                  * We shouldn't need to lock the buffer here, since we flagged
4719                  * it as ARC_L2_WRITING in the previous step, but we must take
4720                  * care to only access its L2 cache parameters. In particular,
4721                  * ab->b_buf may be invalid by now due to ARC eviction.
4722                  */
4723                 l2hdr = ab->b_l2hdr;
4724                 l2hdr->b_daddr = dev->l2ad_hand;
4725 
4726                 if ((ab->b_flags & ARC_L2COMPRESS) &&
4727                     l2hdr->b_asize >= buf_compress_minsz) {
4728                         if (l2arc_compress_buf(l2hdr)) {
4729                                 /*
4730                                  * If compression succeeded, enable headroom
4731                                  * boost on the next scan cycle.
4732                                  */
4733                                 *headroom_boost = B_TRUE;
4734                         }
4735                 }
4736 
4737                 /*
4738                  * Pick up the buffer data we had previously stashed away
4739                  * (and now potentially also compressed).
4740                  */
4741                 buf_data = l2hdr->b_tmp_cdata;
4742                 buf_sz = l2hdr->b_asize;
4743 
4744                 /* Compression may have squashed the buffer to zero length. */
4745                 if (buf_sz != 0) {
4746                         uint64_t buf_p_sz;
4747 
4748                         wzio = zio_write_phys(pio, dev->l2ad_vdev,
4749                             dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
4750                             NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
4751                             ZIO_FLAG_CANFAIL, B_FALSE);
4752 
4753                         DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
4754                             zio_t *, wzio);
4755                         (void) zio_nowait(wzio);
4756 
4757                         write_asize += buf_sz;
4758                         /*
4759                          * Keep the clock hand suitably device-aligned.
4760                          */
4761                         buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
4762                         write_psize += buf_p_sz;
4763                         dev->l2ad_hand += buf_p_sz;
4764                 }
4765         }
4766 
4767         mutex_exit(&l2arc_buflist_mtx);
4768 
4769         ASSERT3U(write_asize, <=, target_sz);
4770         ARCSTAT_BUMP(arcstat_l2_writes_sent);
4771         ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
4772         ARCSTAT_INCR(arcstat_l2_size, write_sz);
4773         ARCSTAT_INCR(arcstat_l2_asize, write_asize);
4774         vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
4775 
4776         /*
4777          * Bump device hand to the device start if it is approaching the end.
4778          * l2arc_evict() will already have evicted ahead for this case.
4779          */
4780         if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
4781                 vdev_space_update(dev->l2ad_vdev,
4782                     dev->l2ad_end - dev->l2ad_hand, 0, 0);
4783                 dev->l2ad_hand = dev->l2ad_start;
4784                 dev->l2ad_evict = dev->l2ad_start;
4785                 dev->l2ad_first = B_FALSE;
4786         }
4787 
4788         dev->l2ad_writing = B_TRUE;
4789         (void) zio_wait(pio);
4790         dev->l2ad_writing = B_FALSE;
4791 
4792         return (write_asize);
4793 }
4794 
4795 /*
4796  * Compresses an L2ARC buffer.
4797  * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its
4798  * size in l2hdr->b_asize. This routine tries to compress the data and
4799  * depending on the compression result there are three possible outcomes:
4800  * *) The buffer was incompressible. The original l2hdr contents were left
4801  *    untouched and are ready for writing to an L2 device.
4802  * *) The buffer was all-zeros, so there is no need to write it to an L2
4803  *    device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
4804  *    set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
4805  * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
4806  *    data buffer which holds the compressed data to be written, and b_asize
4807  *    tells us how much data there is. b_compress is set to the appropriate
4808  *    compression algorithm. Once writing is done, invoke
4809  *    l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
4810  *
4811  * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
4812  * buffer was incompressible).
4813  */
4814 static boolean_t
4815 l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
4816 {
4817         void *cdata;
4818         size_t csize, len;
4819 
4820         ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF);
4821         ASSERT(l2hdr->b_tmp_cdata != NULL);
4822 
4823         len = l2hdr->b_asize;
4824         cdata = zio_data_buf_alloc(len);
4825         csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata,
4826             cdata, l2hdr->b_asize);
4827 
4828         if (csize == 0) {
4829                 /* zero block, indicate that there's nothing to write */
4830                 zio_data_buf_free(cdata, len);
4831                 l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
4832                 l2hdr->b_asize = 0;
4833                 l2hdr->b_tmp_cdata = NULL;
4834                 ARCSTAT_BUMP(arcstat_l2_compress_zeros);
4835                 return (B_TRUE);
4836         } else if (csize > 0 && csize < len) {
4837                 /*
4838                  * Compression succeeded, we'll keep the cdata around for
4839                  * writing and release it afterwards.
4840                  */
4841                 l2hdr->b_compress = ZIO_COMPRESS_LZ4;
4842                 l2hdr->b_asize = csize;
4843                 l2hdr->b_tmp_cdata = cdata;
4844                 ARCSTAT_BUMP(arcstat_l2_compress_successes);
4845                 return (B_TRUE);
4846         } else {
4847                 /*
4848                  * Compression failed, release the compressed buffer.
4849                  * l2hdr will be left unmodified.
4850                  */
4851                 zio_data_buf_free(cdata, len);
4852                 ARCSTAT_BUMP(arcstat_l2_compress_failures);
4853                 return (B_FALSE);
4854         }
4855 }
4856 
4857 /*
4858  * Decompresses a zio read back from an l2arc device. On success, the
4859  * underlying zio's io_data buffer is overwritten by the uncompressed
4860  * version. On decompression error (corrupt compressed stream), the
4861  * zio->io_error value is set to signal an I/O error.
4862  *
4863  * Please note that the compressed data stream is not checksummed, so
4864  * if the underlying device is experiencing data corruption, we may feed
4865  * corrupt data to the decompressor, so the decompressor needs to be
4866  * able to handle this situation (LZ4 does).
4867  */
4868 static void
4869 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
4870 {
4871         ASSERT(L2ARC_IS_VALID_COMPRESS(c));
4872 
4873         if (zio->io_error != 0) {
4874                 /*
4875                  * An io error has occured, just restore the original io
4876                  * size in preparation for a main pool read.
4877                  */
4878                 zio->io_orig_size = zio->io_size = hdr->b_size;
4879                 return;
4880         }
4881 
4882         if (c == ZIO_COMPRESS_EMPTY) {
4883                 /*
4884                  * An empty buffer results in a null zio, which means we
4885                  * need to fill its io_data after we're done restoring the
4886                  * buffer's contents.
4887                  */
4888                 ASSERT(hdr->b_buf != NULL);
4889                 bzero(hdr->b_buf->b_data, hdr->b_size);
4890                 zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
4891         } else {
4892                 ASSERT(zio->io_data != NULL);
4893                 /*
4894                  * We copy the compressed data from the start of the arc buffer
4895                  * (the zio_read will have pulled in only what we need, the
4896                  * rest is garbage which we will overwrite at decompression)
4897                  * and then decompress back to the ARC data buffer. This way we
4898                  * can minimize copying by simply decompressing back over the
4899                  * original compressed data (rather than decompressing to an
4900                  * aux buffer and then copying back the uncompressed buffer,
4901                  * which is likely to be much larger).
4902                  */
4903                 uint64_t csize;
4904                 void *cdata;
4905 
4906                 csize = zio->io_size;
4907                 cdata = zio_data_buf_alloc(csize);
4908                 bcopy(zio->io_data, cdata, csize);
4909                 if (zio_decompress_data(c, cdata, zio->io_data, csize,
4910                     hdr->b_size) != 0)
4911                         zio->io_error = EIO;
4912                 zio_data_buf_free(cdata, csize);
4913         }
4914 
4915         /* Restore the expected uncompressed IO size. */
4916         zio->io_orig_size = zio->io_size = hdr->b_size;
4917 }
4918 
4919 /*
4920  * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
4921  * This buffer serves as a temporary holder of compressed data while
4922  * the buffer entry is being written to an l2arc device. Once that is
4923  * done, we can dispose of it.
4924  */
4925 static void
4926 l2arc_release_cdata_buf(arc_buf_hdr_t *ab)
4927 {
4928         l2arc_buf_hdr_t *l2hdr = ab->b_l2hdr;
4929 
4930         if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
4931                 /*
4932                  * If the data was compressed, then we've allocated a
4933                  * temporary buffer for it, so now we need to release it.
4934                  */
4935                 ASSERT(l2hdr->b_tmp_cdata != NULL);
4936                 zio_data_buf_free(l2hdr->b_tmp_cdata, ab->b_size);
4937         }
4938         l2hdr->b_tmp_cdata = NULL;
4939 }
4940 
4941 /*
4942  * This thread feeds the L2ARC at regular intervals.  This is the beating
4943  * heart of the L2ARC.
4944  */
4945 static void
4946 l2arc_feed_thread(void)
4947 {
4948         callb_cpr_t cpr;
4949         l2arc_dev_t *dev;
4950         spa_t *spa;
4951         uint64_t size, wrote;
4952         clock_t begin, next = ddi_get_lbolt();
4953         boolean_t headroom_boost = B_FALSE;
4954 
4955         CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
4956 
4957         mutex_enter(&l2arc_feed_thr_lock);
4958 
4959         while (l2arc_thread_exit == 0) {
4960                 CALLB_CPR_SAFE_BEGIN(&cpr);
4961                 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
4962                     next);
4963                 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
4964                 next = ddi_get_lbolt() + hz;
4965 
4966                 /*
4967                  * Quick check for L2ARC devices.
4968                  */
4969                 mutex_enter(&l2arc_dev_mtx);
4970                 if (l2arc_ndev == 0) {
4971                         mutex_exit(&l2arc_dev_mtx);
4972                         continue;
4973                 }
4974                 mutex_exit(&l2arc_dev_mtx);
4975                 begin = ddi_get_lbolt();
4976 
4977                 /*
4978                  * This selects the next l2arc device to write to, and in
4979                  * doing so the next spa to feed from: dev->l2ad_spa.   This
4980                  * will return NULL if there are now no l2arc devices or if
4981                  * they are all faulted.
4982                  *
4983                  * If a device is returned, its spa's config lock is also
4984                  * held to prevent device removal.  l2arc_dev_get_next()
4985                  * will grab and release l2arc_dev_mtx.
4986                  */
4987                 if ((dev = l2arc_dev_get_next()) == NULL)
4988                         continue;
4989 
4990                 spa = dev->l2ad_spa;
4991                 ASSERT(spa != NULL);
4992 
4993                 /*
4994                  * If the pool is read-only then force the feed thread to
4995                  * sleep a little longer.
4996                  */
4997                 if (!spa_writeable(spa)) {
4998                         next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
4999                         spa_config_exit(spa, SCL_L2ARC, dev);
5000                         continue;
5001                 }
5002 
5003                 /*
5004                  * Avoid contributing to memory pressure.
5005                  */
5006                 if (arc_reclaim_needed()) {
5007                         ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
5008                         spa_config_exit(spa, SCL_L2ARC, dev);
5009                         continue;
5010                 }
5011 
5012                 ARCSTAT_BUMP(arcstat_l2_feeds);
5013 
5014                 size = l2arc_write_size();
5015 
5016                 /*
5017                  * Evict L2ARC buffers that will be overwritten.
5018                  */
5019                 l2arc_evict(dev, size, B_FALSE);
5020 
5021                 /*
5022                  * Write ARC buffers.
5023                  */
5024                 wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
5025 
5026                 /*
5027                  * Calculate interval between writes.
5028                  */
5029                 next = l2arc_write_interval(begin, size, wrote);
5030                 spa_config_exit(spa, SCL_L2ARC, dev);
5031         }
5032 
5033         l2arc_thread_exit = 0;
5034         cv_broadcast(&l2arc_feed_thr_cv);
5035         CALLB_CPR_EXIT(&cpr);               /* drops l2arc_feed_thr_lock */
5036         thread_exit();
5037 }
5038 
5039 boolean_t
5040 l2arc_vdev_present(vdev_t *vd)
5041 {
5042         l2arc_dev_t *dev;
5043 
5044         mutex_enter(&l2arc_dev_mtx);
5045         for (dev = list_head(l2arc_dev_list); dev != NULL;
5046             dev = list_next(l2arc_dev_list, dev)) {
5047                 if (dev->l2ad_vdev == vd)
5048                         break;
5049         }
5050         mutex_exit(&l2arc_dev_mtx);
5051 
5052         return (dev != NULL);
5053 }
5054 
5055 /*
5056  * Add a vdev for use by the L2ARC.  By this point the spa has already
5057  * validated the vdev and opened it.
5058  */
5059 void
5060 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
5061 {
5062         l2arc_dev_t *adddev;
5063 
5064         ASSERT(!l2arc_vdev_present(vd));
5065 
5066         /*
5067          * Create a new l2arc device entry.
5068          */
5069         adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
5070         adddev->l2ad_spa = spa;
5071         adddev->l2ad_vdev = vd;
5072         adddev->l2ad_start = VDEV_LABEL_START_SIZE;
5073         adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
5074         adddev->l2ad_hand = adddev->l2ad_start;
5075         adddev->l2ad_evict = adddev->l2ad_start;
5076         adddev->l2ad_first = B_TRUE;
5077         adddev->l2ad_writing = B_FALSE;
5078 
5079         /*
5080          * This is a list of all ARC buffers that are still valid on the
5081          * device.
5082          */
5083         adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
5084         list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
5085             offsetof(arc_buf_hdr_t, b_l2node));
5086 
5087         vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
5088 
5089         /*
5090          * Add device to global list
5091          */
5092         mutex_enter(&l2arc_dev_mtx);
5093         list_insert_head(l2arc_dev_list, adddev);
5094         atomic_inc_64(&l2arc_ndev);
5095         mutex_exit(&l2arc_dev_mtx);
5096 }
5097 
5098 /*
5099  * Remove a vdev from the L2ARC.
5100  */
5101 void
5102 l2arc_remove_vdev(vdev_t *vd)
5103 {
5104         l2arc_dev_t *dev, *nextdev, *remdev = NULL;
5105 
5106         /*
5107          * Find the device by vdev
5108          */
5109         mutex_enter(&l2arc_dev_mtx);
5110         for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
5111                 nextdev = list_next(l2arc_dev_list, dev);
5112                 if (vd == dev->l2ad_vdev) {
5113                         remdev = dev;
5114                         break;
5115                 }
5116         }
5117         ASSERT(remdev != NULL);
5118 
5119         /*
5120          * Remove device from global list
5121          */
5122         list_remove(l2arc_dev_list, remdev);
5123         l2arc_dev_last = NULL;          /* may have been invalidated */
5124         atomic_dec_64(&l2arc_ndev);
5125         mutex_exit(&l2arc_dev_mtx);
5126 
5127         /*
5128          * Clear all buflists and ARC references.  L2ARC device flush.
5129          */
5130         l2arc_evict(remdev, 0, B_TRUE);
5131         list_destroy(remdev->l2ad_buflist);
5132         kmem_free(remdev->l2ad_buflist, sizeof (list_t));
5133         kmem_free(remdev, sizeof (l2arc_dev_t));
5134 }
5135 
5136 void
5137 l2arc_init(void)
5138 {
5139         l2arc_thread_exit = 0;
5140         l2arc_ndev = 0;
5141         l2arc_writes_sent = 0;
5142         l2arc_writes_done = 0;
5143 
5144         mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
5145         cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
5146         mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
5147         mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
5148         mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
5149 
5150         l2arc_dev_list = &L2ARC_dev_list;
5151         l2arc_free_on_write = &L2ARC_free_on_write;
5152         list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
5153             offsetof(l2arc_dev_t, l2ad_node));
5154         list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
5155             offsetof(l2arc_data_free_t, l2df_list_node));
5156 }
5157 
5158 void
5159 l2arc_fini(void)
5160 {
5161         /*
5162          * This is called from dmu_fini(), which is called from spa_fini();
5163          * Because of this, we can assume that all l2arc devices have
5164          * already been removed when the pools themselves were removed.
5165          */
5166 
5167         l2arc_do_free_on_write();
5168 
5169         mutex_destroy(&l2arc_feed_thr_lock);
5170         cv_destroy(&l2arc_feed_thr_cv);
5171         mutex_destroy(&l2arc_dev_mtx);
5172         mutex_destroy(&l2arc_buflist_mtx);
5173         mutex_destroy(&l2arc_free_on_write_mtx);
5174 
5175         list_destroy(l2arc_dev_list);
5176         list_destroy(l2arc_free_on_write);
5177 }
5178 
5179 void
5180 l2arc_start(void)
5181 {
5182         if (!(spa_mode_global & FWRITE))
5183                 return;
5184 
5185         (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
5186             TS_RUN, minclsyspri);
5187 }
5188 
5189 void
5190 l2arc_stop(void)
5191 {
5192         if (!(spa_mode_global & FWRITE))
5193                 return;
5194 
5195         mutex_enter(&l2arc_feed_thr_lock);
5196         cv_signal(&l2arc_feed_thr_cv);      /* kick thread out of startup */
5197         l2arc_thread_exit = 1;
5198         while (l2arc_thread_exit != 0)
5199                 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
5200         mutex_exit(&l2arc_feed_thr_lock);
5201 }