1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/types.h>
27 #include <sys/ksynch.h>
28 #include <sys/cmn_err.h>
29 #include <sys/errno.h>
30 #include <sys/kmem.h>
31 #include <sys/cred.h>
32 #include <sys/buf.h>
33 #include <sys/ddi.h>
34
35 #include <sys/nsc_thread.h>
36 #include <sys/nsctl/nsctl.h>
37
38 #include <sys/sdt.h> /* dtrace is S10 or later */
39
40 #include "sd_bcache.h"
41 #include "sd_trace.h"
42 #include "sd_io.h"
43 #include "sd_bio.h"
44 #include "sd_ft.h"
45 #include "sd_misc.h"
46 #include "sd_pcu.h"
47
48 #include <sys/unistat/spcs_s.h>
49 #include <sys/unistat/spcs_s_k.h>
50 #include <sys/unistat/spcs_errors.h>
51 #include <sys/nsctl/safestore.h>
52 #ifndef DS_DDICT
53 #include <sys/ddi_impldefs.h>
54 #endif
55
56
57 /*
58 * kstat interface
59 */
60
61 static kstat_t *sdbc_global_stats_kstat;
62 static int sdbc_global_stats_update(kstat_t *ksp, int rw);
63
64 typedef struct {
65 kstat_named_t ci_sdbc_count;
66 kstat_named_t ci_sdbc_loc_count;
67 kstat_named_t ci_sdbc_rdhits;
68 kstat_named_t ci_sdbc_rdmiss;
69 kstat_named_t ci_sdbc_wrhits;
70 kstat_named_t ci_sdbc_wrmiss;
71 kstat_named_t ci_sdbc_blksize;
72 kstat_named_t ci_sdbc_lru_blocks;
73 #ifdef DEBUG
74 kstat_named_t ci_sdbc_lru_noreq;
75 kstat_named_t ci_sdbc_lru_req;
76 #endif
77 kstat_named_t ci_sdbc_wlru_inq;
78 kstat_named_t ci_sdbc_cachesize;
79 kstat_named_t ci_sdbc_numblocks;
80 kstat_named_t ci_sdbc_num_shared;
81 kstat_named_t ci_sdbc_wrcancelns;
82 kstat_named_t ci_sdbc_destaged;
83 kstat_named_t ci_sdbc_nodehints;
84 } sdbc_global_stats_t;
85
86 static sdbc_global_stats_t sdbc_global_stats = {
87 {SDBC_GKSTAT_COUNT, KSTAT_DATA_ULONG},
88 {SDBC_GKSTAT_LOC_COUNT, KSTAT_DATA_ULONG},
89 {SDBC_GKSTAT_RDHITS, KSTAT_DATA_ULONG},
90 {SDBC_GKSTAT_RDMISS, KSTAT_DATA_ULONG},
91 {SDBC_GKSTAT_WRHITS, KSTAT_DATA_ULONG},
92 {SDBC_GKSTAT_WRMISS, KSTAT_DATA_ULONG},
93 {SDBC_GKSTAT_BLKSIZE, KSTAT_DATA_ULONG},
94 {SDBC_GKSTAT_LRU_BLOCKS, KSTAT_DATA_ULONG},
95 #ifdef DEBUG
96 {SDBC_GKSTAT_LRU_NOREQ, KSTAT_DATA_ULONG},
97 {SDBC_GKSTAT_LRU_REQ, KSTAT_DATA_ULONG},
98 #endif
99 {SDBC_GKSTAT_WLRU_INQ, KSTAT_DATA_ULONG},
100 {SDBC_GKSTAT_CACHESIZE, KSTAT_DATA_ULONG},
101 {SDBC_GKSTAT_NUMBLOCKS, KSTAT_DATA_ULONG},
102 {SDBC_GKSTAT_NUM_SHARED, KSTAT_DATA_ULONG},
103 {SDBC_GKSTAT_WRCANCELNS, KSTAT_DATA_ULONG},
104 {SDBC_GKSTAT_DESTAGED, KSTAT_DATA_ULONG},
105 {SDBC_GKSTAT_NODEHINTS, KSTAT_DATA_ULONG},
106 };
107
108 static kstat_t **sdbc_cd_kstats;
109 static kstat_t **sdbc_cd_io_kstats;
110 static kmutex_t *sdbc_cd_io_kstats_mutexes;
111 static kstat_t *sdbc_global_io_kstat;
112 static kmutex_t sdbc_global_io_kstat_mutex;
113 static int sdbc_cd_stats_update(kstat_t *ksp, int rw);
114 static int cd_kstat_add(int cd);
115 static int cd_kstat_remove(int cd);
116
117 typedef struct {
118 kstat_named_t ci_sdbc_vol_name;
119 kstat_named_t ci_sdbc_failed;
120 kstat_named_t ci_sdbc_cd;
121 kstat_named_t ci_sdbc_cache_read;
122 kstat_named_t ci_sdbc_cache_write;
123 kstat_named_t ci_sdbc_disk_read;
124 kstat_named_t ci_sdbc_disk_write;
125 kstat_named_t ci_sdbc_filesize;
126 kstat_named_t ci_sdbc_numdirty;
127 kstat_named_t ci_sdbc_numio;
128 kstat_named_t ci_sdbc_numfail;
129 kstat_named_t ci_sdbc_destaged;
130 kstat_named_t ci_sdbc_wrcancelns;
131 kstat_named_t ci_sdbc_cdhints;
132 } sdbc_cd_stats_t;
133
134 static sdbc_cd_stats_t sdbc_cd_stats = {
135 {SDBC_CDKSTAT_VOL_NAME, KSTAT_DATA_CHAR},
136 {SDBC_CDKSTAT_FAILED, KSTAT_DATA_ULONG},
137 {SDBC_CDKSTAT_CD, KSTAT_DATA_ULONG},
138 {SDBC_CDKSTAT_CACHE_READ, KSTAT_DATA_ULONG},
139 {SDBC_CDKSTAT_CACHE_WRITE, KSTAT_DATA_ULONG},
140 {SDBC_CDKSTAT_DISK_READ, KSTAT_DATA_ULONG},
141 {SDBC_CDKSTAT_DISK_WRITE, KSTAT_DATA_ULONG},
142 #ifdef NSC_MULTI_TERABYTE
143 {SDBC_CDKSTAT_FILESIZE, KSTAT_DATA_UINT64},
144 #else
145 {SDBC_CDKSTAT_FILESIZE, KSTAT_DATA_ULONG},
146 #endif
147 {SDBC_CDKSTAT_NUMDIRTY, KSTAT_DATA_ULONG},
148 {SDBC_CDKSTAT_NUMIO, KSTAT_DATA_ULONG},
149 {SDBC_CDKSTAT_NUMFAIL, KSTAT_DATA_ULONG},
150 {SDBC_CDKSTAT_DESTAGED, KSTAT_DATA_ULONG},
151 {SDBC_CDKSTAT_WRCANCELNS, KSTAT_DATA_ULONG},
152 {SDBC_CDKSTAT_CDHINTS, KSTAT_DATA_ULONG},
153 };
154
155 #ifdef DEBUG
156 /*
157 * dynmem kstat interface
158 */
159 static kstat_t *sdbc_dynmem_kstat_dm;
160 static int simplect_dm;
161 static int sdbc_dynmem_kstat_update_dm(kstat_t *ksp, int rw);
162
163 typedef struct {
164 kstat_named_t ci_sdbc_monitor_dynmem;
165 kstat_named_t ci_sdbc_max_dyn_list;
166 kstat_named_t ci_sdbc_cache_aging_ct1;
167 kstat_named_t ci_sdbc_cache_aging_ct2;
168 kstat_named_t ci_sdbc_cache_aging_ct3;
169 kstat_named_t ci_sdbc_cache_aging_sec1;
170 kstat_named_t ci_sdbc_cache_aging_sec2;
171 kstat_named_t ci_sdbc_cache_aging_sec3;
172 kstat_named_t ci_sdbc_cache_aging_pcnt1;
173 kstat_named_t ci_sdbc_cache_aging_pcnt2;
174 kstat_named_t ci_sdbc_max_holds_pcnt;
175
176 kstat_named_t ci_sdbc_alloc_ct;
177 kstat_named_t ci_sdbc_dealloc_ct;
178 kstat_named_t ci_sdbc_history;
179 kstat_named_t ci_sdbc_nodatas;
180 kstat_named_t ci_sdbc_candidates;
181 kstat_named_t ci_sdbc_deallocs;
182 kstat_named_t ci_sdbc_hosts;
183 kstat_named_t ci_sdbc_pests;
184 kstat_named_t ci_sdbc_metas;
185 kstat_named_t ci_sdbc_holds;
186 kstat_named_t ci_sdbc_others;
187 kstat_named_t ci_sdbc_notavail;
188
189 kstat_named_t ci_sdbc_process_directive;
190
191 kstat_named_t ci_sdbc_simplect;
192 } sdbc_dynmem_dm_t;
193
194 static sdbc_dynmem_dm_t sdbc_dynmem_dm = {
195 {SDBC_DMKSTAT_MONITOR_DYNMEM, KSTAT_DATA_ULONG},
196 {SDBC_DMKSTAT_MAX_DYN_LIST, KSTAT_DATA_ULONG},
197 {SDBC_DMKSTAT_CACHE_AGING_CT1, KSTAT_DATA_ULONG},
198 {SDBC_DMKSTAT_CACHE_AGING_CT2, KSTAT_DATA_ULONG},
199 {SDBC_DMKSTAT_CACHE_AGING_CT3, KSTAT_DATA_ULONG},
200 {SDBC_DMKSTAT_CACHE_AGING_SEC1, KSTAT_DATA_ULONG},
201 {SDBC_DMKSTAT_CACHE_AGING_SEC2, KSTAT_DATA_ULONG},
202 {SDBC_DMKSTAT_CACHE_AGING_SEC3, KSTAT_DATA_ULONG},
203 {SDBC_DMKSTAT_CACHE_AGING_PCNT1, KSTAT_DATA_ULONG},
204 {SDBC_DMKSTAT_CACHE_AGING_PCNT2, KSTAT_DATA_ULONG},
205 {SDBC_DMKSTAT_MAX_HOLDS_PCNT, KSTAT_DATA_ULONG},
206 {SDBC_DMKSTAT_ALLOC_CNT, KSTAT_DATA_ULONG},
207 {SDBC_DMKSTAT_DEALLOC_CNT, KSTAT_DATA_ULONG},
208 {SDBC_DMKSTAT_HISTORY, KSTAT_DATA_ULONG},
209 {SDBC_DMKSTAT_NODATAS, KSTAT_DATA_ULONG},
210 {SDBC_DMKSTAT_CANDIDATES, KSTAT_DATA_ULONG},
211 {SDBC_DMKSTAT_DEALLOCS, KSTAT_DATA_ULONG},
212 {SDBC_DMKSTAT_HOSTS, KSTAT_DATA_ULONG},
213 {SDBC_DMKSTAT_PESTS, KSTAT_DATA_ULONG},
214 {SDBC_DMKSTAT_METAS, KSTAT_DATA_ULONG},
215 {SDBC_DMKSTAT_HOLDS, KSTAT_DATA_ULONG},
216 {SDBC_DMKSTAT_OTHERS, KSTAT_DATA_ULONG},
217 {SDBC_DMKSTAT_NOTAVAIL, KSTAT_DATA_ULONG},
218 {SDBC_DMKSTAT_PROCESS_DIRECTIVE, KSTAT_DATA_ULONG},
219 {SDBC_DMKSTAT_SIMPLECT, KSTAT_DATA_ULONG}
220 };
221 #endif
222
223 /* End of dynmem kstats */
224
225 #ifdef DEBUG
226 int *dmchainpull_table; /* dmchain wastage stats */
227 #endif
228
229 /*
230 * dynmem process vars
231 */
232 extern _dm_process_vars_t dynmem_processing_dm;
233
234 /* metadata for volumes */
235 ss_voldata_t *_sdbc_gl_file_info;
236
237 size_t _sdbc_gl_file_info_size;
238
239 /* metadata for cache write blocks */
240 static ss_centry_info_t *_sdbc_gl_centry_info;
241
242 /* wblocks * sizeof(ss_centry_info_t) */
243 static size_t _sdbc_gl_centry_info_size;
244
245 static int _SD_DELAY_QUEUE = 1;
246 static int sdbc_allocb_inuse, sdbc_allocb_lost, sdbc_allocb_hit;
247 static int sdbc_allocb_pageio1, sdbc_allocb_pageio2;
248 static int sdbc_centry_hit, sdbc_centry_inuse, sdbc_centry_lost;
249 static int sdbc_dmchain_not_avail;
250 static int sdbc_allocb_deallocd;
251 static int sdbc_centry_deallocd;
252 static int sdbc_check_cot;
253 static int sdbc_ra_hash; /* 1-block read-ahead fails due to hash hit */
254 static int sdbc_ra_none; /* 1-block read-ahead fails due to "would block" */
255
256
257 /*
258 * Set the following variable to 1 to enable pagelist io mutual
259 * exclusion on all _sd_alloc_buf() operations.
260 *
261 * This is set to ON to prevent front end / back end races between new
262 * NSC_WRTHRU io operations coming in through _sd_alloc_buf(), and
263 * previously written data being flushed out to disk by the sdbc
264 * flusher at the back end.
265 * -- see bugtraq 4287564
266 * -- Simon Crosland, Mon Nov 8 16:34:09 GMT 1999
267 */
268 static int sdbc_pageio_always = 1;
269
270 int sdbc_use_dmchain = 0; /* start time switch for dm chaining */
271 int sdbc_prefetch1 = 1; /* do 1-block read-ahead */
272 /*
273 * if sdbc_static_cache is 1 allocate all cache memory at startup.
274 * deallocate only at shutdown.
275 */
276 int sdbc_static_cache = 1;
277
278 #ifdef DEBUG
279 /*
280 * Pagelist io mutual exclusion debug facility.
281 */
282 #define SDBC_PAGEIO_OFF 0 /* no debug */
283 #define SDBC_PAGEIO_RDEV 1 /* force NSC_PAGEIO for specified dev */
284 #define SDBC_PAGEIO_RAND 2 /* randomly force NSC_PAGEIO */
285 #define SDBC_PAGEIO_ALL 3 /* always force NSC_PAGEIO */
286 static int sdbc_pageio_debug = SDBC_PAGEIO_OFF;
287 static dev_t sdbc_pageio_rdev = (dev_t)-1;
288 #endif
289
290 /*
291 * INF SD cache global data
292 */
293
294 _sd_cd_info_t *_sd_cache_files;
295 _sd_stats_t *_sd_cache_stats;
296 kmutex_t _sd_cache_lock;
297
298 _sd_hash_table_t *_sd_htable;
299 _sd_queue_t _sd_lru_q;
300
301 _sd_cctl_t *_sd_cctl[_SD_CCTL_GROUPS];
302 int _sd_cctl_groupsz;
303
304 _sd_net_t _sd_net_config;
305
306 extern krwlock_t sdbc_queue_lock;
307
308 unsigned int _sd_node_hint;
309
310 #define _SD_LRU_Q (&_sd_lru_q)
311 int BLK_FBAS; /* number of FBA's in a cache block */
312 int CACHE_BLOCK_SIZE; /* size in bytes of a cache block */
313 int CBLOCKS;
314 _sd_bitmap_t BLK_FBA_BITS;
315 static int sdbc_prefetch_valid_cnt;
316 static int sdbc_prefetch_busy_cnt;
317 static int sdbc_prefetch_trailing;
318 static int sdbc_prefetch_deallocd;
319 static int sdbc_prefetch_pageio1;
320 static int sdbc_prefetch_pageio2;
321 static int sdbc_prefetch_hit;
322 static int sdbc_prefetch_lost;
323 static int _sd_prefetch_opt = 1; /* 0 to disable & use _prefetch_sb_vec[] */
324 static nsc_vec_t _prefetch_sb_vec[_SD_MAX_BLKS + 1];
325
326 _sd_bitmap_t _fba_bits[] = {
327 0x0000, 0x0001, 0x0003, 0x0007,
328 0x000f, 0x001f, 0x003f, 0x007f,
329 0x00ff,
330 #if defined(_SD_8K_BLKSIZE)
331 0x01ff, 0x03ff, 0x07ff,
332 0x0fff, 0x1fff, 0x3fff, 0x7fff,
333 0xffff,
334 #endif
335 };
336
337
338 static int _sd_ccsync_cnt = 256;
339 static _sd_cctl_sync_t *_sd_ccent_sync;
340
341 nsc_io_t *sdbc_io;
342
343 #ifdef _MULTI_DATAMODEL
344 _sd_stats32_t *_sd_cache_stats32 = NULL;
345 #endif
346
347
348 #ifdef DEBUG
349 int cmn_level = CE_PANIC;
350 #else
351 int cmn_level = CE_WARN;
352 #endif
353
354 /*
355 * Forward declare all statics that are used before defined to enforce
356 * parameter checking
357 * Some (if not all) of these could be removed if the code were reordered
358 */
359
360 static void _sdbc_stats_deconfigure(void);
361 static int _sdbc_stats_configure(int cblocks);
362 static int _sdbc_lruq_configure(_sd_queue_t *);
363 static void _sdbc_lruq_deconfigure(void);
364 static int _sdbc_mem_configure(int cblocks, spcs_s_info_t kstatus);
365 static void _sdbc_mem_deconfigure(int cblocks);
366 static void _sd_ins_queue(_sd_queue_t *, _sd_cctl_t *centry);
367 static int _sd_flush_cd(int cd);
368 static int _sd_check_buffer_alloc(int cd, nsc_off_t fba_pos, nsc_size_t fba_len,
369 _sd_buf_handle_t **hp);
370 static int _sd_doread(_sd_buf_handle_t *handle, _sd_cctl_t *cc_ent,
371 nsc_off_t fba_pos, nsc_size_t fba_len, int flag);
372 static void _sd_async_read_ea(blind_t xhandle, nsc_off_t fba_pos,
373 nsc_size_t fba_len, int error);
374 static void _sd_async_write_ea(blind_t xhandle, nsc_off_t fba_pos,
375 nsc_size_t fba_len, int error);
376 static void _sd_queue_write(_sd_buf_handle_t *handle, nsc_off_t fba_pos,
377 nsc_size_t fba_len);
378 static int _sd_remote_store(_sd_cctl_t *cc_ent, nsc_off_t fba_pos,
379 nsc_size_t fba_len);
380 static int _sd_copy_direct(_sd_buf_handle_t *handle1, _sd_buf_handle_t *handle2,
381 nsc_off_t fba_pos1, nsc_off_t fba_pos2, nsc_size_t fba_len);
382 static int _sd_sync_write(_sd_buf_handle_t *handle, nsc_off_t fba_pos,
383 nsc_size_t fba_len, int flag);
384 static int _sd_sync_write2(_sd_buf_handle_t *wr_handle, nsc_off_t wr_st_pos,
385 nsc_size_t fba_len, int flag, _sd_buf_handle_t *rd_handle,
386 nsc_off_t rd_st_pos);
387 static int sdbc_fd_attach_cd(blind_t xcd);
388 static int sdbc_fd_detach_cd(blind_t xcd);
389 static int sdbc_fd_flush_cd(blind_t xcd);
390 static int _sdbc_gl_centry_configure(spcs_s_info_t);
391 static int _sdbc_gl_file_configure(spcs_s_info_t);
392 static void _sdbc_gl_centry_deconfigure(void);
393 static void _sdbc_gl_file_deconfigure(void);
394 static int sdbc_doread_prefetch(_sd_cctl_t *cc_ent, nsc_off_t fba_pos,
395 nsc_size_t fba_len);
396 static _sd_bitmap_t update_dirty(_sd_cctl_t *cc_ent, sdbc_cblk_fba_t st_off,
397 sdbc_cblk_fba_t st_len);
398 static int _sd_prefetch_buf(int cd, nsc_off_t fba_pos, nsc_size_t fba_len,
399 int flag, _sd_buf_handle_t *handle, int locked);
400
401 /* dynmem support */
402 static int _sd_setup_category_on_type(_sd_cctl_t *header);
403 static int _sd_setup_mem_chaining(_sd_cctl_t *header, int flag);
404
405 static int sdbc_check_cctl_cot(_sd_cctl_t *);
406
407 static int sdbc_dmqueues_configure();
408 static void sdbc_dmqueues_deconfigure();
409 static _sd_cctl_t *sdbc_get_dmchain(int, int *, int);
410 static int sdbc_dmchain_avail(_sd_cctl_t *);
411 void sdbc_requeue_dmchain(_sd_queue_t *, _sd_cctl_t *, int, int);
412 static void sdbc_ins_dmqueue_back(_sd_queue_t *, _sd_cctl_t *);
413 void sdbc_ins_dmqueue_front(_sd_queue_t *, _sd_cctl_t *);
414 void sdbc_remq_dmchain(_sd_queue_t *, _sd_cctl_t *);
415 static void sdbc_clear_dmchain(_sd_cctl_t *, _sd_cctl_t *);
416 void sdbc_requeue_head_dm_try(_sd_cctl_t *);
417 static _sd_cctl_t *sdbc_alloc_dmc(int, nsc_off_t, nsc_size_t, int *,
418 sdbc_allocbuf_t *, int);
419 static _sd_cctl_t *sdbc_alloc_lru(int, nsc_off_t, int *, int);
420 static _sd_cctl_t *sdbc_alloc_from_dmchain(int, nsc_off_t, sdbc_allocbuf_t *,
421 int);
422 static void sdbc_centry_init_dm(_sd_cctl_t *);
423 static int sdbc_centry_memalloc_dm(_sd_cctl_t *, int, int);
424 static void sdbc_centry_alloc_end(sdbc_allocbuf_t *);
425
426
427
428
429 /* _SD_DEBUG */
430 #if defined(_SD_DEBUG) || defined(DEBUG)
431 static int _sd_cctl_valid(_sd_cctl_t *);
432 #endif
433
434 static
435 nsc_def_t _sdbc_fd_def[] = {
436 "Attach", (uintptr_t)sdbc_fd_attach_cd, 0,
437 "Detach", (uintptr_t)sdbc_fd_detach_cd, 0,
438 "Flush", (uintptr_t)sdbc_fd_flush_cd, 0,
439 0, 0, 0
440 };
441
442
443 /*
444 * _sdbc_cache_configure - initialize cache blocks, queues etc.
445 *
446 * ARGUMENTS:
447 * cblocks - Number of cache blocks
448 *
449 * RETURNS:
450 * 0 on success.
451 * SDBC_EENABLEFAIL or SDBC_EMEMCONFIG on failure.
452 *
453 */
454
455
456
457 int
458 _sdbc_cache_configure(int cblocks, spcs_s_info_t kstatus)
459 {
460 CBLOCKS = cblocks;
461
462 _sd_cache_files = (_sd_cd_info_t *)
463 kmem_zalloc(sdbc_max_devs * sizeof (_sd_cd_info_t),
464 KM_SLEEP);
465
466 if (_sdbc_stats_configure(cblocks))
467 return (SDBC_EENABLEFAIL);
468
469 if (sdbc_use_dmchain) {
470 if (sdbc_dmqueues_configure())
471 return (SDBC_EENABLEFAIL);
472 } else {
473 if (_sdbc_lruq_configure(_SD_LRU_Q))
474 return (SDBC_EENABLEFAIL);
475 }
476
477
478 if (_sdbc_mem_configure(cblocks, kstatus))
479 return (SDBC_EMEMCONFIG);
480
481 CACHE_BLOCK_SIZE = BLK_SIZE(1);
482 BLK_FBAS = FBA_NUM(CACHE_BLOCK_SIZE);
483 BLK_FBA_BITS = _fba_bits[BLK_FBAS];
484
485 sdbc_allocb_pageio1 = 0;
486 sdbc_allocb_pageio2 = 0;
487 sdbc_allocb_hit = 0;
488 sdbc_allocb_inuse = 0;
489 sdbc_allocb_lost = 0;
490 sdbc_centry_inuse = 0;
491 sdbc_centry_lost = 0;
492 sdbc_centry_hit = 0;
493 sdbc_centry_deallocd = 0;
494 sdbc_dmchain_not_avail = 0;
495 sdbc_allocb_deallocd = 0;
496
497 sdbc_prefetch_valid_cnt = 0;
498 sdbc_prefetch_busy_cnt = 0;
499 sdbc_prefetch_trailing = 0;
500 sdbc_prefetch_deallocd = 0;
501 sdbc_prefetch_pageio1 = 0;
502 sdbc_prefetch_pageio2 = 0;
503 sdbc_prefetch_hit = 0;
504 sdbc_prefetch_lost = 0;
505
506 sdbc_check_cot = 0;
507 sdbc_prefetch1 = 1;
508 sdbc_ra_hash = 0;
509 sdbc_ra_none = 0;
510
511 return (0);
512 }
513
514 /*
515 * _sdbc_cache_deconfigure - cache is being deconfigured. Release any
516 * memory that we acquired during the configuration process and return
517 * to the unconfigured state.
518 *
519 * NOTE: all users of the cache should be inactive at this point,
520 * i.e. we are unregistered from sd and all cache daemons/threads are
521 * gone.
522 *
523 */
524 void
525 _sdbc_cache_deconfigure(void)
526 {
527 /* CCIO shutdown must happen before memory is free'd */
528
529 if (_sd_cache_files) {
530 kmem_free(_sd_cache_files,
531 sdbc_max_devs * sizeof (_sd_cd_info_t));
532 _sd_cache_files = (_sd_cd_info_t *)NULL;
533 }
534
535
536 BLK_FBA_BITS = 0;
537 BLK_FBAS = 0;
538 CACHE_BLOCK_SIZE = 0;
539 _sdbc_mem_deconfigure(CBLOCKS);
540 _sdbc_gl_centry_deconfigure();
541 _sdbc_gl_file_deconfigure();
542
543 if (sdbc_use_dmchain)
544 sdbc_dmqueues_deconfigure();
545 else
546 _sdbc_lruq_deconfigure();
547 _sdbc_stats_deconfigure();
548
549 CBLOCKS = 0;
550 }
551
552
553 /*
554 * _sdbc_stats_deconfigure - cache is being deconfigured turn off
555 * stats. This could seemingly do more but we leave most of the
556 * data intact until cache is configured again.
557 *
558 */
559 static void
560 _sdbc_stats_deconfigure(void)
561 {
562 int i;
563
564 #ifdef DEBUG
565 if (sdbc_dynmem_kstat_dm) {
566 kstat_delete(sdbc_dynmem_kstat_dm);
567 sdbc_dynmem_kstat_dm = NULL;
568 }
569 #endif
570
571 if (sdbc_global_stats_kstat) {
572 kstat_delete(sdbc_global_stats_kstat);
573 sdbc_global_stats_kstat = NULL;
574 }
575
576 if (sdbc_cd_kstats) {
577 for (i = 0; i < sdbc_max_devs; i++) {
578 if (sdbc_cd_kstats[i]) {
579 kstat_delete(sdbc_cd_kstats[i]);
580 sdbc_cd_kstats[i] = NULL;
581 }
582 }
583 kmem_free(sdbc_cd_kstats, sizeof (kstat_t *) * sdbc_max_devs);
584 sdbc_cd_kstats = NULL;
585 }
586
587 if (sdbc_global_io_kstat) {
588 kstat_delete(sdbc_global_io_kstat);
589 mutex_destroy(&sdbc_global_io_kstat_mutex);
590 sdbc_global_io_kstat = NULL;
591 }
592
593 if (sdbc_cd_io_kstats) {
594 for (i = 0; i < sdbc_max_devs; i++) {
595 if (sdbc_cd_io_kstats[i]) {
596 kstat_delete(sdbc_cd_io_kstats[i]);
597 sdbc_cd_io_kstats[i] = NULL;
598 }
599 }
600 kmem_free(sdbc_cd_io_kstats, sizeof (kstat_t *) *
601 sdbc_max_devs);
602 sdbc_cd_io_kstats = NULL;
603 }
604
605 if (sdbc_cd_io_kstats_mutexes) {
606 /* mutexes are already destroyed in cd_kstat_remove() */
607 kmem_free(sdbc_cd_io_kstats_mutexes,
608 sizeof (kmutex_t) * sdbc_max_devs);
609 sdbc_cd_io_kstats_mutexes = NULL;
610 }
611
612
613 if (_sd_cache_stats) {
614 kmem_free(_sd_cache_stats,
615 sizeof (_sd_stats_t) +
616 (sdbc_max_devs - 1) * sizeof (_sd_shared_t));
617 _sd_cache_stats = NULL;
618 }
619 #ifdef _MULTI_DATAMODEL
620 if (_sd_cache_stats32) {
621 kmem_free(_sd_cache_stats32, sizeof (_sd_stats32_t) +
622 (sdbc_max_devs - 1) * sizeof (_sd_shared_t));
623 _sd_cache_stats32 = NULL;
624 }
625 #endif
626 }
627
628 static int
629 _sdbc_stats_configure(int cblocks)
630 {
631
632 _sd_cache_stats = kmem_zalloc(sizeof (_sd_stats_t) +
633 (sdbc_max_devs - 1) * sizeof (_sd_shared_t), KM_SLEEP);
634 _sd_cache_stats->st_blksize = (int)BLK_SIZE(1);
635 _sd_cache_stats->st_cachesize = cblocks * BLK_SIZE(1);
636 _sd_cache_stats->st_numblocks = cblocks;
637 _sd_cache_stats->st_wrcancelns = 0;
638 _sd_cache_stats->st_destaged = 0;
639 #ifdef _MULTI_DATAMODEL
640 _sd_cache_stats32 = kmem_zalloc(sizeof (_sd_stats32_t) +
641 (sdbc_max_devs - 1) * sizeof (_sd_shared_t), KM_SLEEP);
642 #endif
643
644 /* kstat implementation - global stats */
645 sdbc_global_stats_kstat = kstat_create(SDBC_KSTAT_MODULE, 0,
646 SDBC_KSTAT_GSTATS, SDBC_KSTAT_CLASS, KSTAT_TYPE_NAMED,
647 sizeof (sdbc_global_stats)/sizeof (kstat_named_t),
648 KSTAT_FLAG_VIRTUAL|KSTAT_FLAG_WRITABLE);
649
650 if (sdbc_global_stats_kstat != NULL) {
651 sdbc_global_stats_kstat->ks_data = &sdbc_global_stats;
652 sdbc_global_stats_kstat->ks_update = sdbc_global_stats_update;
653 sdbc_global_stats_kstat->ks_private = _sd_cache_stats;
654 kstat_install(sdbc_global_stats_kstat);
655 } else {
656 cmn_err(CE_WARN, "!sdbc: gstats kstat failed");
657 }
658
659 /* global I/O kstats */
660 sdbc_global_io_kstat = kstat_create(SDBC_KSTAT_MODULE, 0,
661 SDBC_IOKSTAT_GSTATS, "disk", KSTAT_TYPE_IO, 1, 0);
662
663 if (sdbc_global_io_kstat) {
664 mutex_init(&sdbc_global_io_kstat_mutex, NULL, MUTEX_DRIVER,
665 NULL);
666 sdbc_global_io_kstat->ks_lock =
667 &sdbc_global_io_kstat_mutex;
668 kstat_install(sdbc_global_io_kstat);
669 }
670
671 /*
672 * kstat implementation - cd stats
673 * NOTE: one kstat instance for each open cache descriptor
674 */
675 sdbc_cd_kstats = kmem_zalloc(sizeof (kstat_t *) * sdbc_max_devs,
676 KM_SLEEP);
677
678 /*
679 * kstat implementation - i/o kstats per cache descriptor
680 * NOTE: one I/O kstat instance for each cd
681 */
682 sdbc_cd_io_kstats = kmem_zalloc(sizeof (kstat_t *) * sdbc_max_devs,
683 KM_SLEEP);
684
685 sdbc_cd_io_kstats_mutexes = kmem_zalloc(sizeof (kmutex_t) *
686 sdbc_max_devs, KM_SLEEP);
687
688 #ifdef DEBUG
689 /* kstat implementation - dynamic memory stats */
690 sdbc_dynmem_kstat_dm = kstat_create(SDBC_KSTAT_MODULE, 0,
691 SDBC_KSTAT_DYNMEM, SDBC_KSTAT_CLASS, KSTAT_TYPE_NAMED,
692 sizeof (sdbc_dynmem_dm)/sizeof (kstat_named_t),
693 KSTAT_FLAG_VIRTUAL|KSTAT_FLAG_WRITABLE);
694
695 if (sdbc_dynmem_kstat_dm != NULL) {
696 sdbc_dynmem_kstat_dm->ks_data = &sdbc_dynmem_dm;
697 sdbc_dynmem_kstat_dm->ks_update = sdbc_dynmem_kstat_update_dm;
698 sdbc_dynmem_kstat_dm->ks_private = &dynmem_processing_dm;
699 kstat_install(sdbc_dynmem_kstat_dm);
700 } else {
701 cmn_err(CE_WARN, "!sdbc: dynmem kstat failed");
702 }
703 #endif
704
705 return (0);
706 }
707
708 /*
709 * sdbc_dmqueues_configure()
710 * initialize the queues of dynamic memory chains.
711 */
712
713 _sd_queue_t *sdbc_dm_queues;
714 static int max_dm_queues;
715
716
717 static int
718 sdbc_dmqueues_configure()
719 {
720 int i;
721
722 /*
723 * CAUTION! this code depends on max_dyn_list not changing
724 * if it does change behavior may be incorrect, as cc_alloc_size_dm
725 * depends on max_dyn_list and indexes to dmqueues are derived from
726 * cc_alloc_size_dm.
727 * see _sd_setup_category_on_type() and _sd_dealloc_dm()
728 * TODO: prevent max_dyn_list from on-the-fly modification (easy) or
729 * allow for on-the-fly changes to number of dm queues (hard).
730 */
731 max_dm_queues = dynmem_processing_dm.max_dyn_list;
732
733 ++max_dm_queues; /* need a "0" queue for centrys with no memory */
734
735 sdbc_dm_queues = (_sd_queue_t *)
736 kmem_zalloc(max_dm_queues * sizeof (_sd_queue_t), KM_SLEEP);
737
738 #ifdef DEBUG
739 dmchainpull_table = (int *)kmem_zalloc(max_dm_queues *
740 max_dm_queues * sizeof (int), KM_SLEEP);
741 #endif
742
743 for (i = 0; i < max_dm_queues; ++i) {
744 (void) _sdbc_lruq_configure(&sdbc_dm_queues[i]);
745 sdbc_dm_queues[i].sq_dmchain_cblocks = i;
746 }
747
748 return (0);
749 }
750
751 static void
752 sdbc_dmqueues_deconfigure()
753 {
754 /* CAUTION! this code depends on max_dyn_list not changing */
755
756 if (sdbc_dm_queues)
757 kmem_free(sdbc_dm_queues, max_dm_queues * sizeof (_sd_queue_t));
758 sdbc_dm_queues = NULL;
759 max_dm_queues = 0;
760 }
761
762 #define GOOD_LRUSIZE(q) ((q->sq_inq >= 0) || (q->sq_inq <= CBLOCKS))
763
764 /*
765 * _sdbc_lruq_configure - initialize the lru queue
766 *
767 * ARGUMENTS: NONE
768 * RETURNS: 0
769 *
770 */
771
772 static int
773 _sdbc_lruq_configure(_sd_queue_t *_sd_lru)
774 {
775
776 _sd_lru->sq_inq = 0;
777
778 mutex_init(&_sd_lru->sq_qlock, NULL, MUTEX_DRIVER, NULL);
779
780 _sd_lru->sq_qhead.cc_next = _sd_lru->sq_qhead.cc_prev
781 = &(_sd_lru->sq_qhead);
782 return (0);
783 }
784
785 /*
786 * _sdbc_lruq_deconfigure - deconfigure the lru queue
787 *
788 * ARGUMENTS: NONE
789 *
790 */
791
792 static void
793 _sdbc_lruq_deconfigure(void)
794 {
795 _sd_queue_t *_sd_lru;
796
797 _sd_lru = _SD_LRU_Q;
798
799 mutex_destroy(&_sd_lru->sq_qlock);
800 bzero(_sd_lru, sizeof (_sd_queue_t));
801
802 }
803
804 /*
805 * _sdbc_mem_configure - initialize the cache memory.
806 * Create and initialize the hash table.
807 * Create cache control blocks and fill them with relevent
808 * information and enqueue onto the lru queue.
809 * Initialize the Write control blocks (blocks that contain
810 * information as to where the data will be mirrored)
811 * Initialize the Fault tolerant blocks (blocks that contain
812 * information about the mirror nodes dirty writes)
813 *
814 * ARGUMENTS:
815 * cblocks - Number of cache blocks.
816 * RETURNS: 0
817 *
818 */
819 static int
820 _sdbc_mem_configure(int cblocks, spcs_s_info_t kstatus)
821 {
822 int num_blks, i, blk;
823 _sd_cctl_t *centry;
824 _sd_net_t *netc;
825 _sd_cctl_t *prev_entry_dm, *first_entry_dm;
826
827 if ((_sd_htable = _sdbc_hash_configure(cblocks)) == NULL) {
828 spcs_s_add(kstatus, SDBC_ENOHASH);
829 return (-1);
830 }
831
832 _sd_cctl_groupsz = (cblocks / _SD_CCTL_GROUPS) +
833 ((cblocks % _SD_CCTL_GROUPS) != 0);
834
835 for (i = 0; i < _SD_CCTL_GROUPS; i++) {
836 _sd_cctl[i] = (_sd_cctl_t *)
837 nsc_kmem_zalloc(_sd_cctl_groupsz * sizeof (_sd_cctl_t),
838 KM_SLEEP, sdbc_cache_mem);
839
840 if (_sd_cctl[i] == NULL) {
841 spcs_s_add(kstatus, SDBC_ENOCB);
842 return (-1);
843 }
844 }
845
846 _sd_ccent_sync = (_sd_cctl_sync_t *)
847 nsc_kmem_zalloc(_sd_ccsync_cnt * sizeof (_sd_cctl_sync_t),
848 KM_SLEEP, sdbc_local_mem);
849
850 if (_sd_ccent_sync == NULL) {
851 spcs_s_add(kstatus, SDBC_ENOCCTL);
852 return (-1);
853 }
854
855 for (i = 0; i < _sd_ccsync_cnt; i++) {
856 mutex_init(&_sd_ccent_sync[i]._cc_lock, NULL, MUTEX_DRIVER,
857 NULL);
858 cv_init(&_sd_ccent_sync[i]._cc_blkcv, NULL, CV_DRIVER, NULL);
859 }
860
861 blk = 0;
862
863 netc = &_sd_net_config;
864
865 num_blks = (netc->sn_cpages * (int)netc->sn_psize)/BLK_SIZE(1);
866
867 prev_entry_dm = 0;
868 first_entry_dm = 0;
869 for (i = 0; i < num_blks; i++, blk++) {
870 centry = _sd_cctl[(blk/_sd_cctl_groupsz)] +
871 (blk%_sd_cctl_groupsz);
872 centry->cc_sync = &_sd_ccent_sync[blk % _sd_ccsync_cnt];
873 centry->cc_next = centry->cc_prev = NULL;
874 centry->cc_dirty_next = centry->cc_dirty_link = NULL;
875 centry->cc_await_use = centry->cc_await_page = 0;
876 centry->cc_inuse = centry->cc_pageio = 0;
877 centry->cc_flag = 0;
878 centry->cc_iocount = 0;
879 centry->cc_valid = 0;
880
881 if (!first_entry_dm)
882 first_entry_dm = centry;
883 if (prev_entry_dm)
884 prev_entry_dm->cc_link_list_dm = centry;
885 prev_entry_dm = centry;
886 centry->cc_link_list_dm = first_entry_dm;
887 centry->cc_data = 0;
888 centry->cc_write = NULL;
889 centry->cc_dirty = 0;
890
891 {
892 _sd_queue_t *q;
893 if (sdbc_use_dmchain) {
894 q = &sdbc_dm_queues[0];
895 centry->cc_cblocks = 0;
896 } else
897 q = _SD_LRU_Q;
898
899 _sd_ins_queue(q, centry);
900 }
901
902 }
903
904 if (_sdbc_gl_centry_configure(kstatus) != 0)
905 return (-1);
906
907 if (_sdbc_gl_file_configure(kstatus) != 0)
908 return (-1);
909
910 return (0);
911 }
912
913 /*
914 * _sdbc_gl_file_configure()
915 * allocate and initialize space for the global filename data.
916 *
917 */
918 static int
919 _sdbc_gl_file_configure(spcs_s_info_t kstatus)
920 {
921 ss_voldata_t *fileinfo;
922 ss_voldata_t tempfinfo;
923 ss_vdir_t vdir;
924 ss_vdirkey_t key;
925 int err = 0;
926
927 _sdbc_gl_file_info_size = safestore_config.ssc_maxfiles *
928 sizeof (ss_voldata_t);
929
930 if ((_sdbc_gl_file_info = kmem_zalloc(_sdbc_gl_file_info_size,
931 KM_NOSLEEP)) == NULL) {
932 spcs_s_add(kstatus, SDBC_ENOSFNV);
933 return (-1);
934 }
935
936 /* setup the key to get a directory stream of all volumes */
937 key.vk_type = CDIR_ALL;
938
939 fileinfo = _sdbc_gl_file_info;
940
941 /*
942 * if coming up after a crash, "refresh" the host
943 * memory copy from safestore.
944 */
945 if (_sdbc_warm_start()) {
946
947 if (SSOP_GETVDIR(sdbc_safestore, &key, &vdir)) {
948 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_file_configure): "
949 "cannot read safestore");
950 return (-1);
951 }
952
953
954 /*
955 * cycle through the vdir getting volume data
956 * and volume tokens
957 */
958
959 while ((err = SSOP_GETVDIRENT(sdbc_safestore, &vdir, fileinfo))
960 == SS_OK) {
961 ++fileinfo;
962 }
963
964 if (err != SS_EOF) {
965 /*
966 * fail to configure since
967 * recovery is not possible.
968 */
969 spcs_s_add(kstatus, SDBC_ENOREFRESH);
970 return (-1);
971 }
972
973 } else { /* normal initialization, not a warm start */
974
975 /*
976 * if this fails, continue: cache will start
977 * in writethru mode
978 */
979
980 if (SSOP_GETVDIR(sdbc_safestore, &key, &vdir)) {
981 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_file_configure): "
982 "cannot read safestore");
983 return (-1);
984 }
985
986 /*
987 * cycle through the vdir getting just the volume tokens
988 * and initializing volume entries
989 */
990
991 while ((err = SSOP_GETVDIRENT(sdbc_safestore, &vdir,
992 &tempfinfo)) == 0) {
993 /*
994 * initialize the host memory copy of the
995 * global file region. this means setting the
996 * _pinned and _attached fields to _SD_NO_HOST
997 * because the default of zero conflicts with
998 * the min nodeid of zero.
999 */
1000 fileinfo->sv_vol = tempfinfo.sv_vol;
1001 fileinfo->sv_pinned = _SD_NO_HOST;
1002 fileinfo->sv_attached = _SD_NO_HOST;
1003 fileinfo->sv_cd = _SD_NO_CD;
1004
1005 /* initialize the directory entry */
1006 if ((err = SSOP_SETVOL(sdbc_safestore, fileinfo))
1007 == SS_ERR) {
1008 cmn_err(CE_WARN,
1009 "!sdbc(_sdbc_gl_file_configure): "
1010 "volume entry write failure %p",
1011 (void *)fileinfo->sv_vol);
1012 break;
1013 }
1014
1015 ++fileinfo;
1016 }
1017
1018 /* coming up clean, continue in w-t mode */
1019 if (err != SS_EOF)
1020 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_file_configure) "
1021 "unable to init safe store volinfo");
1022 }
1023
1024 return (0);
1025 }
1026
1027 static void
1028 _sdbc_gl_centry_deconfigure(void)
1029 {
1030 if (_sdbc_gl_centry_info)
1031 kmem_free(_sdbc_gl_centry_info, _sdbc_gl_centry_info_size);
1032
1033 _sdbc_gl_centry_info = NULL;
1034 _sdbc_gl_centry_info_size = 0;
1035 }
1036
1037 static int
1038 _sdbc_gl_centry_configure(spcs_s_info_t kstatus)
1039 {
1040
1041 int wblocks;
1042 ss_centry_info_t *cinfo;
1043 ss_cdirkey_t key;
1044 ss_cdir_t cdir;
1045 int err = 0;
1046
1047
1048 wblocks = safestore_config.ssc_wsize / BLK_SIZE(1);
1049 _sdbc_gl_centry_info_size = sizeof (ss_centry_info_t) * wblocks;
1050
1051 if ((_sdbc_gl_centry_info = kmem_zalloc(_sdbc_gl_centry_info_size,
1052 KM_NOSLEEP)) == NULL) {
1053 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_centry_configure) "
1054 "alloc failed for gl_centry_info region");
1055
1056 _sdbc_gl_centry_deconfigure();
1057 return (-1);
1058 }
1059
1060 /*
1061 * synchronize the centry info area with safe store
1062 */
1063
1064 /* setup the key to get a directory stream of all centrys */
1065 key.ck_type = CDIR_ALL;
1066
1067 cinfo = _sdbc_gl_centry_info;
1068
1069 if (_sdbc_warm_start()) {
1070
1071 if (SSOP_GETCDIR(sdbc_safestore, &key, &cdir)) {
1072 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_centry_configure): "
1073 "cannot read safestore");
1074 return (-1);
1075 }
1076
1077
1078 /*
1079 * cycle through the cdir getting resource
1080 * tokens and reading centrys
1081 */
1082
1083 while ((err = SSOP_GETCDIRENT(sdbc_safestore, &cdir, cinfo))
1084 == 0) {
1085 ++cinfo;
1086 }
1087
1088 if (err != SS_EOF) {
1089 /*
1090 * fail to configure since
1091 * recovery is not possible.
1092 */
1093 _sdbc_gl_centry_deconfigure();
1094 spcs_s_add(kstatus, SDBC_EGLDMAFAIL);
1095 return (-1);
1096 }
1097
1098 } else {
1099
1100 if (SSOP_GETCDIR(sdbc_safestore, &key, &cdir)) {
1101 cmn_err(CE_WARN, "!sdbc(_sdbc_gl_centry_configure): "
1102 "cannot read safestore");
1103 return (-1);
1104 }
1105
1106 /*
1107 * cycle through the cdir getting resource
1108 * tokens and initializing centrys
1109 */
1110
1111 while ((err = SSOP_GETCDIRENT(sdbc_safestore, &cdir, cinfo))
1112 == 0) {
1113 cinfo->sc_cd = -1;
1114 cinfo->sc_fpos = -1;
1115
1116 if ((err = SSOP_SETCENTRY(sdbc_safestore, cinfo))
1117 == SS_ERR) {
1118 cmn_err(CE_WARN,
1119 "!sdbc(_sdbc_gl_centry_configure): "
1120 "cache entry write failure %p",
1121 (void *)cinfo->sc_res);
1122 break;
1123 }
1124
1125 ++cinfo;
1126 }
1127
1128 /* coming up clean, continue in w-t mode */
1129 if (err != SS_EOF) {
1130 cmn_err(CE_WARN, "!sdbc(sdbc_gl_centry_configure) "
1131 "_sdbc_gl_centry_info initialization failed");
1132 }
1133 }
1134
1135 return (0);
1136 }
1137
1138
1139 static void
1140 _sdbc_gl_file_deconfigure(void)
1141 {
1142
1143 if (_sdbc_gl_file_info)
1144 kmem_free(_sdbc_gl_file_info, _sdbc_gl_file_info_size);
1145
1146 _sdbc_gl_file_info = NULL;
1147
1148 _sdbc_gl_file_info_size = 0;
1149 }
1150
1151
1152 /*
1153 * _sdbc_mem_deconfigure - deconfigure the cache memory.
1154 * Release any memory/locks/sv's acquired during _sdbc_mem_configure.
1155 *
1156 * ARGUMENTS:
1157 * cblocks - Number of cache blocks.
1158 *
1159 */
1160 /* ARGSUSED */
1161 static void
1162 _sdbc_mem_deconfigure(int cblocks)
1163 {
1164 int i;
1165
1166 if (_sd_ccent_sync) {
1167 for (i = 0; i < _sd_ccsync_cnt; i++) {
1168 mutex_destroy(&_sd_ccent_sync[i]._cc_lock);
1169 cv_destroy(&_sd_ccent_sync[i]._cc_blkcv);
1170 }
1171 nsc_kmem_free(_sd_ccent_sync,
1172 _sd_ccsync_cnt * sizeof (_sd_cctl_sync_t));
1173 }
1174 _sd_ccent_sync = NULL;
1175
1176 for (i = 0; i < _SD_CCTL_GROUPS; i++) {
1177 if (_sd_cctl[i] != NULL) {
1178 nsc_kmem_free(_sd_cctl[i],
1179 _sd_cctl_groupsz * sizeof (_sd_cctl_t));
1180 _sd_cctl[i] = NULL;
1181 }
1182 }
1183 _sd_cctl_groupsz = 0;
1184
1185 _sdbc_hash_deconfigure(_sd_htable);
1186 _sd_htable = NULL;
1187
1188 }
1189
1190
1191 #if defined(_SD_DEBUG) || defined(DEBUG)
1192 static int
1193 _sd_cctl_valid(_sd_cctl_t *addr)
1194 {
1195 _sd_cctl_t *end;
1196 int i, valid;
1197
1198 valid = 0;
1199 for (i = 0; i < _SD_CCTL_GROUPS; i++) {
1200 end = _sd_cctl[i] + _sd_cctl_groupsz;
1201 if (addr >= _sd_cctl[i] && addr < end) {
1202 valid = 1;
1203 break;
1204 }
1205 }
1206
1207 return (valid);
1208 }
1209 #endif
1210
1211
1212 /*
1213 * _sd_ins_queue - insert centry into LRU queue
1214 * (during initialization, locking not required)
1215 */
1216 static void
1217 _sd_ins_queue(_sd_queue_t *q, _sd_cctl_t *centry)
1218 {
1219 _sd_cctl_t *q_head;
1220
1221 ASSERT(_sd_cctl_valid(centry));
1222
1223 q_head = &q->sq_qhead;
1224 centry->cc_prev = q_head;
1225 centry->cc_next = q_head->cc_next;
1226 q_head->cc_next->cc_prev = centry;
1227 q_head->cc_next = centry;
1228 q->sq_inq++;
1229
1230 ASSERT(GOOD_LRUSIZE(q));
1231 }
1232
1233
1234
1235 void
1236 _sd_requeue(_sd_cctl_t *centry)
1237 {
1238 _sd_queue_t *q = _SD_LRU_Q;
1239
1240 /* was FAST */
1241 mutex_enter(&q->sq_qlock);
1242 #if defined(_SD_DEBUG)
1243 if (1) {
1244 _sd_cctl_t *cp, *cn, *qp;
1245 cp = centry->cc_prev;
1246 cn = centry->cc_next;
1247 qp = (q->sq_qhead).cc_prev;
1248 if (!_sd_cctl_valid(centry) ||
1249 (cp != &(q->sq_qhead) && !_sd_cctl_valid(cp)) ||
1250 (cn != &(q->sq_qhead) && !_sd_cctl_valid(cn)) ||
1251 !_sd_cctl_valid(qp))
1252 cmn_err(CE_PANIC,
1253 "_sd_requeue %x prev %x next %x qp %x",
1254 centry, cp, cn, qp);
1255 }
1256 #endif
1257 centry->cc_prev->cc_next = centry->cc_next;
1258 centry->cc_next->cc_prev = centry->cc_prev;
1259 centry->cc_next = &(q->sq_qhead);
1260 centry->cc_prev = q->sq_qhead.cc_prev;
1261 q->sq_qhead.cc_prev->cc_next = centry;
1262 q->sq_qhead.cc_prev = centry;
1263 centry->cc_seq = q->sq_seq++;
1264 /* was FAST */
1265 mutex_exit(&q->sq_qlock);
1266 (q->sq_req_stat)++;
1267
1268 }
1269
1270 void
1271 _sd_requeue_head(_sd_cctl_t *centry)
1272 {
1273 _sd_queue_t *q = _SD_LRU_Q;
1274
1275 /* was FAST */
1276 mutex_enter(&q->sq_qlock);
1277 #if defined(_SD_DEBUG)
1278 if (1) {
1279 _sd_cctl_t *cp, *cn, *qn;
1280 cp = centry->cc_prev;
1281 cn = centry->cc_next;
1282 qn = (q->sq_qhead).cc_prev;
1283 if (!_sd_cctl_valid(centry) ||
1284 (cp != &(q->sq_qhead) && !_sd_cctl_valid(cp)) ||
1285 (cn != &(q->sq_qhead) && !_sd_cctl_valid(cn)) ||
1286 !_sd_cctl_valid(qn))
1287 cmn_err(CE_PANIC,
1288 "_sd_requeue_head %x prev %x next %x qn %x",
1289 centry, cp, cn, qn);
1290 }
1291 #endif
1292 centry->cc_prev->cc_next = centry->cc_next;
1293 centry->cc_next->cc_prev = centry->cc_prev;
1294 centry->cc_prev = &(q->sq_qhead);
1295 centry->cc_next = q->sq_qhead.cc_next;
1296 q->sq_qhead.cc_next->cc_prev = centry;
1297 q->sq_qhead.cc_next = centry;
1298 centry->cc_seq = q->sq_seq++;
1299 centry->cc_flag &= ~CC_QHEAD;
1300 /* was FAST */
1301 mutex_exit(&q->sq_qlock);
1302 }
1303
1304
1305
1306 /*
1307 * _sd_open - Open a file.
1308 *
1309 * ARGUMENTS:
1310 * filename - Name of the file to be opened.
1311 * flag - Flag associated with open.
1312 * (currently used to determine a ckd device)
1313 * RETURNS:
1314 * cd - the cache descriptor.
1315 */
1316
1317 int
1318 _sd_open(char *filename, int flag)
1319 {
1320 int cd;
1321
1322 if (!_sd_cache_initialized) {
1323 cmn_err(CE_WARN, "!sdbc(_sd_open) cache not initialized");
1324 return (-EINVAL);
1325 }
1326 cd = _sd_open_cd(filename, -1, flag);
1327 SDTRACE(SDF_OPEN, (cd < 0) ? SDT_INV_CD : cd, 0, SDT_INV_BL, 0, cd);
1328
1329 return (cd);
1330 }
1331
1332
1333 static int
1334 _sd_open_io(char *filename, int flag, blind_t *cdp, nsc_iodev_t *iodev)
1335 {
1336 _sd_cd_info_t *cdi;
1337 int cd;
1338 int rc = 0;
1339
1340 if ((cd = _sd_open(filename, flag)) >= 0) {
1341
1342 cdi = &(_sd_cache_files[cd]);
1343 cdi->cd_iodev = iodev;
1344 nsc_set_owner(cdi->cd_rawfd, cdi->cd_iodev);
1345
1346 *cdp = (blind_t)(unsigned long)cd;
1347 } else
1348 rc = -cd;
1349
1350 return (rc);
1351 }
1352
1353
1354
1355 int
1356 _sd_open_cd(char *filename, const int cd, const int flag)
1357 {
1358 int new_cd, rc = 0, alloc_cd = -1;
1359 ss_voldata_t *cdg;
1360 int preexists = 0;
1361 _sd_cd_info_t *cdi;
1362 int failover_open, open_failed;
1363 major_t devmaj;
1364 minor_t devmin;
1365
1366 if (_sdbc_shutdown_in_progress)
1367 return (-EIO);
1368
1369 if (strlen(filename) > (NSC_MAXPATH-1))
1370 return (-ENAMETOOLONG);
1371
1372 /*
1373 * If the cd is >= 0, then this is a open for a specific cd.
1374 * This happens when the mirror node crashes, and we attempt to
1375 * reopen the files with the same cache descriptors as existed on
1376 * the other node
1377 */
1378
1379 retry_open:
1380 failover_open = 0;
1381 open_failed = 0;
1382 if (cd >= 0) {
1383 failover_open++;
1384 cdi = &(_sd_cache_files[cd]);
1385 mutex_enter(&_sd_cache_lock);
1386 if (cdi->cd_info == NULL)
1387 cdi->cd_info = &_sd_cache_stats->st_shared[cd];
1388 else if (cdi->cd_info->sh_alloc &&
1389 strcmp(cdi->cd_info->sh_filename, filename)) {
1390 cmn_err(CE_WARN, "!sdbc(_sd_open_cd) cd %d mismatch",
1391 cd);
1392 mutex_exit(&_sd_cache_lock);
1393 return (-EEXIST);
1394 }
1395
1396 if (cdi->cd_info->sh_failed != 2) {
1397 if (cdi->cd_info->sh_alloc != 0)
1398 preexists = 1;
1399 else {
1400 cdi->cd_info->sh_alloc = CD_ALLOC_IN_PROGRESS;
1401 (void) strcpy(cdi->cd_info->sh_filename,
1402 filename);
1403 if (_sd_cache_stats->st_count < sdbc_max_devs)
1404 _sd_cache_stats->st_count++;
1405 }
1406 }
1407
1408 mutex_exit(&_sd_cache_lock);
1409 alloc_cd = cd;
1410
1411 goto known_cd;
1412 }
1413
1414 new_cd = 0;
1415 mutex_enter(&_sd_cache_lock);
1416
1417 for (cdi = &(_sd_cache_files[new_cd]),
1418 cdg = _sdbc_gl_file_info + new_cd;
1419 new_cd < (sdbc_max_devs); new_cd++, cdi++, cdg++) {
1420 if (strlen(cdg->sv_volname) != 0)
1421 if (strcmp(cdg->sv_volname, filename))
1422 continue;
1423
1424 if (cdi->cd_info == NULL)
1425 cdi->cd_info = &_sd_cache_stats->st_shared[new_cd];
1426
1427 if (cdi->cd_info->sh_failed != 2) {
1428 if (cdi->cd_info->sh_alloc != 0)
1429 preexists = 1;
1430 else {
1431 if (cd == -2) {
1432 mutex_exit(&_sd_cache_lock);
1433 return (-1);
1434 }
1435 cdi->cd_info->sh_alloc = CD_ALLOC_IN_PROGRESS;
1436 (void) strcpy(cdi->cd_info->sh_filename,
1437 filename);
1438 (void) strcpy(cdg->sv_volname, filename);
1439
1440 cdg->sv_cd = new_cd;
1441 /* update safestore */
1442 SSOP_SETVOL(sdbc_safestore, cdg);
1443 if (_sd_cache_stats->st_count < sdbc_max_devs)
1444 _sd_cache_stats->st_count++;
1445 cdi->cd_flag = 0;
1446 }
1447 }
1448 alloc_cd = new_cd;
1449 break;
1450 }
1451
1452 mutex_exit(&_sd_cache_lock);
1453
1454 if (alloc_cd == -1)
1455 return (-ENOSPC);
1456
1457 known_cd:
1458 /*
1459 * If preexists: someone else is attempting to open this file as
1460 * well. Do only one open, but block everyone else here till the
1461 * open is completed.
1462 */
1463 if (preexists) {
1464 while (cdi->cd_info->sh_alloc == CD_ALLOC_IN_PROGRESS) {
1465 delay(drv_usectohz(20000));
1466 }
1467 if ((cdi->cd_info->sh_alloc != CD_ALLOCATED))
1468 goto retry_open;
1469 return (alloc_cd);
1470 }
1471
1472 if (!(cdi->cd_rawfd =
1473 nsc_open(filename, NSC_SDBC_ID|NSC_DEVICE, _sdbc_fd_def,
1474 (blind_t)(unsigned long)alloc_cd, &rc)) ||
1475 !nsc_getval(cdi->cd_rawfd, "DevMaj", (int *)&devmaj) ||
1476 !nsc_getval(cdi->cd_rawfd, "DevMin", (int *)&devmin)) {
1477 if (cdi->cd_rawfd) {
1478 (void) nsc_close(cdi->cd_rawfd);
1479 cdi->cd_rawfd = NULL;
1480 }
1481 /*
1482 * take into account that there may be pinned data on a
1483 * device that can no longer be opened
1484 */
1485 open_failed++;
1486 if (!(cdi->cd_info->sh_failed) && !failover_open) {
1487 cdi->cd_info->sh_alloc = 0;
1488 mutex_enter(&_sd_cache_lock);
1489 _sd_cache_stats->st_count--;
1490 mutex_exit(&_sd_cache_lock);
1491 if (!rc)
1492 rc = EIO;
1493 return (-rc);
1494 }
1495 }
1496
1497 cdi->cd_strategy = nsc_get_strategy(devmaj);
1498 cdi->cd_crdev = makedevice(devmaj, devmin);
1499 cdi->cd_desc = alloc_cd;
1500 cdi->cd_dirty_head = cdi->cd_dirty_tail = NULL;
1501 cdi->cd_io_head = cdi->cd_io_tail = NULL;
1502 cdi->cd_hint = 0;
1503 #ifdef DEBUG
1504 /* put the dev_t in the ioerr_inject_table */
1505 _sdbc_ioj_set_dev(alloc_cd, cdi->cd_crdev);
1506 #endif
1507
1508 cdi->cd_global = (_sdbc_gl_file_info + alloc_cd);
1509 if (open_failed) {
1510 cdi->cd_info->sh_failed = 2;
1511 } else if (cdi->cd_info->sh_failed != 2)
1512 if ((cdi->cd_global->sv_pinned == _SD_SELF_HOST) &&
1513 !failover_open)
1514 cdi->cd_info->sh_failed = 1;
1515 else
1516 cdi->cd_info->sh_failed = 0;
1517
1518 cdi->cd_flag |= flag;
1519 mutex_init(&cdi->cd_lock, NULL, MUTEX_DRIVER, NULL);
1520
1521 #ifndef _SD_NOTRACE
1522 (void) _sdbc_tr_configure(alloc_cd);
1523 #endif
1524 cdi->cd_info->sh_alloc = CD_ALLOCATED;
1525 cdi->cd_global = (_sdbc_gl_file_info + alloc_cd);
1526 cdi->cd_info->sh_cd = (unsigned short) alloc_cd;
1527 mutex_enter(&_sd_cache_lock);
1528 _sd_cache_stats->st_loc_count++;
1529 mutex_exit(&_sd_cache_lock);
1530
1531 if (cd_kstat_add(alloc_cd) < 0) {
1532 cmn_err(CE_WARN, "!Could not create kstats for cache descriptor"
1533 " %d", alloc_cd);
1534 }
1535
1536
1537 return (open_failed ? -EIO : alloc_cd);
1538 }
1539
1540
1541 /*
1542 * _sd_close - Close a cache descriptor.
1543 *
1544 * ARGUMENTS:
1545 * cd - the cache descriptor to be closed.
1546 * RETURNS:
1547 * 0 on success.
1548 * Error otherwise.
1549 *
1550 * Note: Under Construction.
1551 */
1552
1553 int
1554 _sd_close(int cd)
1555 {
1556 int rc;
1557 _sd_cd_info_t *cdi = &(_sd_cache_files[cd]);
1558
1559 if (!FILE_OPENED(cd)) {
1560 rc = EINVAL;
1561 goto out;
1562 }
1563
1564 SDTRACE(ST_ENTER|SDF_CLOSE, cd, 0, SDT_INV_BL, 0, 0);
1565
1566 mutex_enter(&_sd_cache_lock);
1567 if ((cdi->cd_info->sh_alloc == 0) ||
1568 (cdi->cd_info->sh_alloc & CD_CLOSE_IN_PROGRESS)) {
1569 mutex_exit(&_sd_cache_lock);
1570 SDTRACE(ST_EXIT|SDF_CLOSE, cd, 0, SDT_INV_BL, 0, EINVAL);
1571 rc = EINVAL;
1572 goto out;
1573 }
1574 cdi->cd_info->sh_alloc |= CD_CLOSE_IN_PROGRESS;
1575 mutex_exit(&_sd_cache_lock);
1576
1577 /*
1578 * _sd_flush_cd() will return -1 for the case where pinned
1579 * data is present, but has been transfered to the mirror
1580 * node. In this case it is safe to close the device as
1581 * though _sd_flush_cd() had returned 0.
1582 */
1583
1584 rc = _sd_flush_cd(cd);
1585 if (rc == -1)
1586 rc = 0;
1587
1588 if (rc != 0) {
1589 mutex_enter(&_sd_cache_lock);
1590 if ((rc == EAGAIN) &&
1591 (cdi->cd_global->sv_pinned == _SD_NO_HOST)) {
1592 cdi->cd_global->sv_pinned = _SD_SELF_HOST;
1593 SSOP_SETVOL(sdbc_safestore, cdi->cd_global);
1594 }
1595
1596 cdi->cd_info->sh_alloc &= ~CD_CLOSE_IN_PROGRESS;
1597 mutex_exit(&_sd_cache_lock);
1598 SDTRACE(ST_EXIT|SDF_CLOSE, cd, 0, SDT_INV_BL,
1599 _SD_CD_WBLK_USED(cd), rc);
1600 goto out;
1601 }
1602
1603 rc = nsc_close(cdi->cd_rawfd);
1604 if (rc) {
1605 mutex_enter(&_sd_cache_lock);
1606 cdi->cd_info->sh_alloc &= ~CD_CLOSE_IN_PROGRESS;
1607 mutex_exit(&_sd_cache_lock);
1608 SDTRACE(ST_EXIT|SDF_CLOSE, cd, 0, SDT_INV_BL, 0, rc);
1609 goto out;
1610 }
1611 mutex_enter(&_sd_cache_lock);
1612 _sd_cache_stats->st_loc_count--;
1613 mutex_exit(&_sd_cache_lock);
1614
1615 if (cd_kstat_remove(cd) < 0) {
1616 cmn_err(CE_WARN, "!Could not remove kstat for cache descriptor "
1617 "%d", cd);
1618 }
1619
1620 cdi->cd_info->sh_alloc = 0;
1621 cdi->cd_info->sh_failed = 0;
1622 /* cdi->cd_info = NULL; */
1623 cdi->cd_flag = 0;
1624 SDTRACE(ST_EXIT|SDF_CLOSE, cd, 0, SDT_INV_BL, 0, NSC_DONE);
1625 rc = NSC_DONE;
1626 goto out;
1627
1628 out:
1629 return (rc);
1630 }
1631
1632
1633 static int
1634 _sd_close_io(blind_t xcd)
1635 {
1636 _sd_cd_info_t *cdi;
1637 int cd = (int)(unsigned long)xcd;
1638 int rc = 0;
1639
1640 if ((rc = _sd_close((int)cd)) == NSC_DONE) {
1641 cdi = &(_sd_cache_files[cd]);
1642 cdi->cd_iodev = NULL;
1643 }
1644
1645 return (rc);
1646 }
1647
1648
1649 /*
1650 * _sdbc_remote_store_pinned - reflect pinned/failed blocks for cd
1651 * to our remote mirror. Returns count of blocks reflected or -1 on error.
1652 *
1653 */
1654 int
1655 _sdbc_remote_store_pinned(int cd)
1656 {
1657 int cnt = 0;
1658 _sd_cd_info_t *cdi = &(_sd_cache_files[cd]);
1659 _sd_cctl_t *cc_ent, *cc_list;
1660
1661 ASSERT(cd >= 0);
1662 if (cdi->cd_info->sh_failed) {
1663
1664 if (cdi->cd_global->sv_pinned == _SD_NO_HOST) {
1665 cdi->cd_global->sv_pinned = _SD_SELF_HOST;
1666 SSOP_SETVOL(sdbc_safestore, cdi->cd_global);
1667 }
1668
1669 mutex_enter(&cdi->cd_lock);
1670 cc_ent = cc_list = cdi->cd_fail_head;
1671 while (cc_ent) {
1672 cnt++;
1673
1674 /* is this always necessary? jgk */
1675
1676 if (SSOP_WRITE_CBLOCK(sdbc_safestore,
1677 cc_ent->cc_write->sc_res, cc_ent->cc_data,
1678 CACHE_BLOCK_SIZE, 0)) {
1679 mutex_exit(&cdi->cd_lock);
1680 return (-1);
1681 }
1682
1683 /* update the cache block metadata */
1684 CENTRY_SET_FTPOS(cc_ent);
1685 cc_ent->cc_write->sc_flag = cc_ent->cc_flag;
1686
1687 cc_ent->cc_write->sc_dirty = CENTRY_DIRTY(cc_ent);
1688
1689 SSOP_SETCENTRY(sdbc_safestore, cc_ent->cc_write);
1690
1691 cc_ent = cc_ent->cc_dirty_next;
1692 if (!cc_ent)
1693 cc_ent = cc_list = cc_list->cc_dirty_link;
1694 }
1695 mutex_exit(&cdi->cd_lock);
1696 }
1697
1698 return (cnt);
1699 }
1700
1701 /*
1702 * _sd_flush_cd()
1703 * reflect pinned blocks to mirrored node
1704 * wait for dirty blocks to be flushed
1705 * returns:
1706 * EIO I/O failure, or pinned blocks and no mirror
1707 * EAGAIN Hang: count of outstanding writes isn't decreasing
1708 * -1 pinned blocks, reflected to mirror
1709 * 0 success
1710 */
1711 static int
1712 _sd_flush_cd(int cd)
1713 {
1714 int rc;
1715
1716 if ((rc = _sd_wait_for_flush(cd)) == 0)
1717 return (0);
1718
1719 /*
1720 * if we timed out simply return otherwise
1721 * it must be an i/o type of error
1722 */
1723 if (rc == EAGAIN)
1724 return (rc);
1725
1726 if (_sd_is_mirror_down())
1727 return (EIO); /* already failed, no mirror */
1728
1729 /* flush any pinned/failed blocks to mirror */
1730 if (_sdbc_remote_store_pinned(cd) >= 0)
1731 /*
1732 * At this point it looks like we have blocks on the
1733 * failed list and taking up space on this node but
1734 * no longer have responsibility for the blocks.
1735 * These blocks will in fact be freed from the cache
1736 * and the failed list when the mirror picks them up
1737 * from safe storage and then calls _sd_cd_discard_mirror
1738 * which will issue an rpc telling us to finish up.
1739 *
1740 * Should the other node die before sending the rpc then
1741 * we are safe with these blocks simply waiting on the
1742 * failed list.
1743 */
1744 return (-1);
1745 else
1746 return (rc);
1747 }
1748
1749 /*
1750 * _sdbc_io_attach_cd -- set up for client access to device, reserve raw device
1751 *
1752 * ARGUMENTS:
1753 * cd - the cache descriptor to attach.
1754 *
1755 * RETURNS:
1756 * 0 on success.
1757 * Error otherwise.
1758 */
1759 int
1760 _sdbc_io_attach_cd(blind_t xcd)
1761 {
1762 int rc = 0;
1763 _sd_cd_info_t *cdi;
1764 int cd = (int)(unsigned long)xcd;
1765
1766 SDTRACE(ST_ENTER|SDF_ATTACH, cd, 0, SDT_INV_BL, 0, 0);
1767 if (!_sd_cache_initialized ||
1768 _sdbc_shutdown_in_progress ||
1769 !FILE_OPENED(cd)) {
1770 SDTRACE(ST_EXIT|SDF_ATTACH, cd, 0, SDT_INV_BL, 0, EINVAL);
1771
1772 DTRACE_PROBE(_sdbc_io_attach_cd_end1);
1773
1774 return (EINVAL);
1775 }
1776 cdi = &(_sd_cache_files[cd]);
1777
1778 /*
1779 * check if disk is failed without raw device open. If it is,
1780 * it has to be recovered using _sd_disk_online
1781 */
1782
1783 if (cdi->cd_global->sv_pinned == _SD_SELF_HOST) {
1784 _sd_print(3,
1785 "_sdbc_io_attach_cd: pinned data. returning EINVAL");
1786
1787 DTRACE_PROBE(_sdbc_io_attach_cd_end2);
1788
1789 return (EINVAL);
1790 }
1791
1792 if ((cdi->cd_info == NULL) || (cdi->cd_info->sh_failed)) {
1793 DTRACE_PROBE1(_sdbc_io_attach_cd_end3,
1794 struct _sd_shared *, cdi->cd_info);
1795
1796 return (EINVAL);
1797 }
1798
1799 #if defined(_SD_FAULT_RES)
1800 /* wait for node recovery to finish */
1801 if (_sd_node_recovery)
1802 (void) _sd_recovery_wait();
1803 #endif
1804
1805 /* this will provoke a sdbc_fd_attach_cd call .. */
1806
1807 rc = nsc_reserve(cdi->cd_rawfd, NSC_MULTI);
1808 SDTRACE(ST_EXIT|SDF_ATTACH, cd, 0, SDT_INV_BL, 0, rc);
1809
1810 return (rc);
1811 }
1812
1813 /*
1814 * sdbc_fd_attach_cd -- setup cache for access to raw device underlying cd.
1815 * This is provoked by some piece of sdbc doing a reserve on the raw device.
1816 *
1817 * ARGUMENTS:
1818 * cd - the cache descriptor to attach.
1819 *
1820 * RETURNS:
1821 * 0 on success.
1822 * Error otherwise.
1823 */
1824 static int
1825 sdbc_fd_attach_cd(blind_t xcd)
1826 {
1827 int rc = 0;
1828 int cd = (int)(unsigned long)xcd;
1829 _sd_cd_info_t *cdi;
1830
1831 if (!_sd_cache_initialized || !FILE_OPENED(cd)) {
1832 SDTRACE(ST_INFO|SDF_ATTACH, cd, 0, SDT_INV_BL, 0, EINVAL);
1833
1834 DTRACE_PROBE(sdbc_fd_attach_cd_end1);
1835
1836 return (EINVAL);
1837 }
1838 cdi = &(_sd_cache_files[cd]);
1839
1840 #if defined(_SD_FAULT_RES)
1841 /* retrieve pinned/failed data */
1842 if (!_sd_node_recovery) {
1843 (void) _sd_repin_cd(cd);
1844 }
1845 #endif
1846
1847 rc = nsc_partsize(cdi->cd_rawfd, &cdi->cd_info->sh_filesize);
1848 if (rc != 0) {
1849 SDTRACE(ST_INFO|SDF_ATTACH, cd, 0, SDT_INV_BL, 0, rc);
1850
1851 DTRACE_PROBE(sdbc_fd_attach_cd_end3);
1852
1853 return (rc);
1854 }
1855
1856 cdi->cd_global->sv_attached = _SD_SELF_HOST;
1857
1858 SSOP_SETVOL(sdbc_safestore, cdi->cd_global);
1859
1860 mutex_enter(&_sd_cache_lock);
1861 cdi->cd_info->sh_flag |= CD_ATTACHED;
1862 mutex_exit(&_sd_cache_lock);
1863
1864 return (0);
1865 }
1866
1867 /*
1868 * _sdbc_io_detach_cd -- release raw device
1869 * Called when a cache client is being detached from this cd.
1870 *
1871 * ARGUMENTS:
1872 * cd - the cache descriptor to detach.
1873 * RETURNS:
1874 * 0 on success.
1875 * Error otherwise.
1876 */
1877 int
1878 _sdbc_io_detach_cd(blind_t xcd)
1879 {
1880 int cd = (int)(unsigned long)xcd;
1881 _sd_cd_info_t *cdi;
1882
1883
1884 SDTRACE(ST_ENTER|SDF_DETACH, cd, 0, SDT_INV_BL, 0, 0);
1885 if (!_sd_cache_initialized || !FILE_OPENED(cd)) {
1886 SDTRACE(ST_EXIT|SDF_DETACH, cd, 0, SDT_INV_BL, 0, EINVAL);
1887
1888 DTRACE_PROBE(_sdbc_io_detach_cd_end1);
1889
1890 return (EINVAL);
1891 }
1892
1893 #if defined(_SD_FAULT_RES)
1894 if (_sd_node_recovery)
1895 (void) _sd_recovery_wait();
1896 #endif
1897 /* relinquish responsibility for device */
1898 cdi = &(_sd_cache_files[cd]);
1899 if (!(cdi->cd_rawfd) || !nsc_held(cdi->cd_rawfd)) {
1900 cmn_err(CE_WARN, "!sdbc(_sdbc_detach_cd)(%d) not attached", cd);
1901 SDTRACE(ST_EXIT|SDF_DETACH, cd, 0, SDT_INV_BL, 0, EPROTO);
1902 DTRACE_PROBE1(_sdbc_io_detach_cd_end2,
1903 nsc_fd_t *, cdi->cd_rawfd);
1904
1905 return (EPROTO);
1906 }
1907 /* this will provoke/allow a call to sdbc_fd_detach_cd */
1908 nsc_release(cdi->cd_rawfd);
1909
1910 SDTRACE(ST_EXIT|SDF_DETACH, cd, 0, SDT_INV_BL, 0, 0);
1911
1912 return (0);
1913 }
1914
1915 /*
1916 * _sdbc_detach_cd -- flush dirty writes to disk, release raw device
1917 * Called when raw device is being detached from this cd.
1918 *
1919 * ARGUMENTS:
1920 * cd - the cache descriptor to detach.
1921 * rd_only - non-zero if detach is for read access.
1922 * RETURNS:
1923 * 0 on success.
1924 * Error otherwise.
1925 */
1926 static int
1927 sdbc_detach_cd(blind_t xcd, int rd_only)
1928 {
1929 int rc;
1930 int cd = (int)(unsigned long)xcd;
1931 _sd_cd_info_t *cdi;
1932
1933 SDTRACE(ST_INFO|SDF_DETACH, cd, 0, SDT_INV_BL, 0, 0);
1934
1935 if (!_sd_cache_initialized || !FILE_OPENED(cd)) {
1936 SDTRACE(ST_INFO|SDF_DETACH, cd, 0, SDT_INV_BL, 0, EINVAL);
1937
1938 DTRACE_PROBE(sdbc_detach_cd_end1);
1939
1940 return (EINVAL);
1941 }
1942
1943
1944 rc = _sd_flush_cd(cd);
1945 if (rc > 0) {
1946 SDTRACE(ST_INFO|SDF_DETACH, cd, 0, SDT_INV_BL, 0, rc);
1947
1948 DTRACE_PROBE(sdbc_detach_cd_end2);
1949
1950 return (rc);
1951 }
1952
1953 if (!rd_only) {
1954 _sd_hash_invalidate_cd(cd);
1955 cdi = &(_sd_cache_files[cd]);
1956
1957 if (cdi->cd_global->sv_attached == _SD_SELF_HOST) {
1958 cdi->cd_global->sv_attached = _SD_NO_HOST;
1959 SSOP_SETVOL(sdbc_safestore, cdi->cd_global);
1960 } else {
1961 cmn_err(CE_WARN,
1962 "!sdbc(_sdbc_detach_cd) (%d) attached by node %d",
1963 cd, cdi->cd_global->sv_attached);
1964 SDTRACE(SDF_DETACH, cd, 0, SDT_INV_BL, 0, EPROTO);
1965
1966 DTRACE_PROBE1(sdbc_detach_cd_end3,
1967 int, cdi->cd_global->sv_attached);
1968
1969 return (EPROTO);
1970 }
1971
1972 mutex_enter(&_sd_cache_lock);
1973 cdi->cd_info->sh_flag &= ~CD_ATTACHED;
1974 mutex_exit(&_sd_cache_lock);
1975 }
1976
1977 SDTRACE(ST_INFO|SDF_DETACH, cd, 0, SDT_INV_BL, 0, 0);
1978
1979 return (0);
1980 }
1981
1982 /*
1983 * _sdbc_fd_detach_cd -- flush dirty writes to disk, release raw device
1984 * Called when raw device is being detached from this cd.
1985 *
1986 * ARGUMENTS:
1987 * xcd - the cache descriptor to detach.
1988 * RETURNS:
1989 * 0 on success.
1990 * Error otherwise.
1991 */
1992 static int
1993 sdbc_fd_detach_cd(blind_t xcd)
1994 {
1995 return (sdbc_detach_cd(xcd, 0));
1996 }
1997
1998 /*
1999 * sdbc_fd_flush_cd - raw device "xcd" is being detached and needs
2000 * flushing. We only need to flush we don't need to hash invalidate
2001 * this file.
2002 */
2003 static int
2004 sdbc_fd_flush_cd(blind_t xcd)
2005 {
2006 return (sdbc_detach_cd(xcd, 1));
2007 }
2008
2009 /*
2010 * _sd_get_pinned - re-issue PINNED callbacks for cache device
2011 *
2012 * ARGUMENTS:
2013 * cd - the cache descriptor to reissue pinned calbacks from.
2014 * RETURNS:
2015 * 0 on success.
2016 * Error otherwise.
2017 */
2018 int
2019 _sd_get_pinned(blind_t xcd)
2020 {
2021 _sd_cd_info_t *cdi;
2022 _sd_cctl_t *cc_list, *cc_ent;
2023 int cd = (int)(unsigned long)xcd;
2024
2025 cdi = &_sd_cache_files[cd];
2026
2027 if (cd < 0 || cd >= sdbc_max_devs) {
2028 DTRACE_PROBE(_sd_get_pinned_end1);
2029 return (EINVAL);
2030 }
2031
2032 if (!FILE_OPENED(cd)) {
2033 DTRACE_PROBE(_sd_get_pinned_end2);
2034 return (0);
2035 }
2036
2037 mutex_enter(&cdi->cd_lock);
2038
2039 if (!cdi->cd_info->sh_failed) {
2040 mutex_exit(&cdi->cd_lock);
2041
2042 DTRACE_PROBE(_sd_get_pinned_end3);
2043 return (0);
2044 }
2045
2046 cc_ent = cc_list = cdi->cd_fail_head;
2047 while (cc_ent) {
2048 if (CENTRY_PINNED(cc_ent))
2049 nsc_pinned_data(cdi->cd_iodev,
2050 BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)), BLK_FBAS);
2051 cc_ent = cc_ent->cc_dirty_next;
2052 if (!cc_ent)
2053 cc_ent = cc_list = cc_list->cc_dirty_link;
2054 }
2055
2056 mutex_exit(&cdi->cd_lock);
2057
2058 return (0);
2059 }
2060
2061 /*
2062 * _sd_allocate_buf - allocate a vector of buffers for io.
2063 * *This call has been replaced by _sd_alloc_buf*
2064 */
2065
2066 _sd_buf_handle_t *
2067 _sd_allocate_buf(int cd, nsc_off_t fba_pos, nsc_size_t fba_len, int flag,
2068 int *sts)
2069 {
2070 _sd_buf_handle_t *handle = NULL;
2071
2072 *sts = _sd_alloc_buf((blind_t)(unsigned long)cd, fba_pos, fba_len,
2073 flag, &handle);
2074 if (*sts == NSC_HIT)
2075 *sts = NSC_DONE;
2076 return (handle);
2077 }
2078
2079
2080 /*
2081 * _sd_prefetch_buf - _sd_alloc_buf w/flag = NSC_RDAHEAD|NSC_RDBUF
2082 * no 'bufvec' (data is not read by caller)
2083 * skip leading valid or busy entries (data available sooner)
2084 * truncate on busy block (to avoid deadlock)
2085 * release trailing valid entries, adjust length before starting I/O.
2086 */
2087 static int
2088 _sd_prefetch_buf(int cd, nsc_off_t fba_pos, nsc_size_t fba_len, int flag,
2089 _sd_buf_handle_t *handle, int locked)
2090 {
2091 _sd_cd_info_t *cdi;
2092 nsc_off_t cblk; /* position of temp cache block */
2093 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
2094 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
2095 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
2096 nsc_off_t io_pos; /* offset in FBA's */
2097 nsc_size_t fba_orig_len;
2098 int sts, stall;
2099 _sd_cctl_t *centry = NULL;
2100 _sd_cctl_t *lentry = NULL;
2101 _sd_cctl_t *ioent = NULL;
2102 _sd_cctl_t *last_ioent = NULL;
2103 sdbc_allocbuf_t alloc_tok = {0};
2104 int this_entry_type = 0;
2105 nsc_size_t request_blocks = 0; /* number of cache blocks required */
2106 int pageio;
2107
2108 handle->bh_flag |= NSC_HACTIVE;
2109 ASSERT(cd >= 0);
2110 cdi = &_sd_cache_files[cd];
2111
2112 /* prefetch: truncate if req'd */
2113 if (fba_len > sdbc_max_fbas)
2114 fba_len = sdbc_max_fbas;
2115 if ((fba_pos + fba_len) > cdi->cd_info->sh_filesize) {
2116 if (fba_pos >= cdi->cd_info->sh_filesize) {
2117 sts = EIO;
2118 goto done;
2119 }
2120 fba_len = cdi->cd_info->sh_filesize - fba_pos;
2121 }
2122
2123 fba_orig_len = fba_len;
2124
2125 _SD_SETUP_HANDLE(handle, cd, fba_pos, fba_len, flag);
2126 handle->bh_centry = NULL;
2127
2128 cblk = FBA_TO_BLK_NUM(fba_pos);
2129 st_cblk_off = BLK_FBA_OFF(fba_pos);
2130 st_cblk_len = BLK_FBAS - st_cblk_off;
2131
2132 /*
2133 * count number of blocks on chain that is required
2134 */
2135 if ((nsc_size_t)st_cblk_len >= fba_len) {
2136 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
2137 end_cblk_len = 0;
2138 } else {
2139 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
2140 }
2141
2142 request_blocks = 1; /* at least one */
2143
2144 /* middle piece */
2145 request_blocks += (fba_len - (st_cblk_len + end_cblk_len)) >>
2146 BLK_FBA_SHFT;
2147
2148 if (end_cblk_len)
2149 ++request_blocks;
2150
2151 stall = 0;
2152 do {
2153 pageio = ((flag & NSC_PAGEIO) != 0 || sdbc_pageio_always != 0);
2154 cget:
2155 if (centry = (_sd_cctl_t *)
2156 _sd_hash_search(cd, cblk, _sd_htable)) {
2157 try:
2158 /* prefetch: skip leading valid blocks */
2159 if ((ioent == NULL) &&
2160 SDBC_VALID_BITS(st_cblk_off, st_cblk_len, centry)) {
2161 skip:
2162 sdbc_prefetch_valid_cnt++;
2163 --request_blocks;
2164 lentry = centry;
2165 centry = NULL;
2166 cblk++;
2167 fba_len -= st_cblk_len;
2168 st_cblk_off = 0;
2169 st_cblk_len = (sdbc_cblk_fba_t)
2170 ((fba_len > (nsc_size_t)BLK_FBAS) ?
2171 BLK_FBAS : fba_len);
2172 continue;
2173 }
2174
2175 if (SET_CENTRY_INUSE(centry)) {
2176 /*
2177 * prefetch: skip leading busy
2178 * or truncate at busy block
2179 */
2180 if (ioent == NULL)
2181 goto skip;
2182 sdbc_prefetch_busy_cnt++;
2183 fba_orig_len -= fba_len;
2184 fba_len = 0;
2185 centry = lentry; /* backup */
2186 break;
2187 }
2188
2189 /*
2190 * bug 4529671
2191 * now that we own the centry make sure that
2192 * it is still good. it could have been processed
2193 * by _sd_dealloc_dm() in the window between
2194 * _sd_hash_search() and SET_CENTRY_INUSE().
2195 */
2196 if ((_sd_cctl_t *)
2197 _sd_hash_search(cd, cblk, _sd_htable) != centry) {
2198 sdbc_prefetch_deallocd++;
2199 #ifdef DEBUG
2200 cmn_err(CE_WARN,
2201 "!prefetch centry %p cd %d cblk %" NSC_SZFMT
2202 " fba_len %" NSC_SZFMT " lost to dealloc?! "
2203 "cc_data %p",
2204 (void *)centry, cd, cblk, fba_orig_len,
2205 (void *)centry->cc_data);
2206 #endif
2207
2208 CLEAR_CENTRY_INUSE(centry);
2209 continue;
2210 }
2211
2212 if (CC_CD_BLK_MATCH(cd, cblk, centry)) {
2213 /*
2214 * Do pagelist io mutual exclusion
2215 * before messing with the centry.
2216 */
2217 if (pageio && SET_CENTRY_PAGEIO(centry)) {
2218 /* flusher not done with pageio */
2219 /*
2220 * prefetch: skip leading busy
2221 * or truncate at busy block
2222 */
2223 CLEAR_CENTRY_INUSE(centry);
2224 if (ioent == NULL)
2225 goto skip;
2226 sdbc_prefetch_pageio1++;
2227 fba_orig_len -= fba_len;
2228 fba_len = 0;
2229 centry = lentry; /* backup */
2230 break;
2231
2232 }
2233
2234 sdbc_prefetch_hit++;
2235 this_entry_type = HASH_ENTRY_DM;
2236 pageio = 0;
2237 centry->cc_toflush = 0;
2238
2239 centry->cc_hits++;
2240
2241 /* this will reset the age flag */
2242 sdbc_centry_init_dm(centry);
2243
2244 DTRACE_PROBE1(_sd_prefetch_buf,
2245 _sd_cctl_t *, centry);
2246 } else {
2247 /* block mismatch */
2248 sdbc_prefetch_lost++;
2249
2250 CLEAR_CENTRY_INUSE(centry);
2251 continue;
2252 }
2253 } else {
2254 centry = sdbc_centry_alloc(cd, cblk, request_blocks,
2255 &stall, &alloc_tok, ALLOC_NOWAIT);
2256
2257 if (centry == NULL) {
2258 /*
2259 * prefetch: cache is very busy. just do
2260 * the i/o for the blocks already acquired,
2261 * if any.
2262 */
2263 fba_orig_len -= fba_len;
2264 fba_len = 0;
2265 /*
2266 * if we have a chain of centry's
2267 * then back up (set centry to lentry).
2268 * if there is no chain (ioent == NULL)
2269 * then centry remains NULL. this can occur
2270 * if all previous centrys were hash hits
2271 * on valid blocks that were processed in
2272 * the skip logic above.
2273 */
2274 if (ioent)
2275 centry = lentry; /* backup */
2276 break;
2277 }
2278
2279 /*
2280 * dmchaining adjustment.
2281 * if centry was obtained from the dmchain
2282 * then clear local pageio variable because the
2283 * centry already has cc_pageio set.
2284 */
2285 if (CENTRY_PAGEIO(centry))
2286 pageio = 0;
2287
2288 DTRACE_PROBE1(_sd_alloc_buf, _sd_cctl_t *, centry);
2289
2290 this_entry_type = ELIGIBLE_ENTRY_DM;
2291 if (centry->cc_aging_dm & FOUND_IN_HASH_DM)
2292 this_entry_type = HASH_ENTRY_DM;
2293 else {
2294 if (centry->cc_aging_dm & FOUND_HOLD_OVER_DM)
2295 this_entry_type = HOLD_ENTRY_DM;
2296 }
2297 }
2298
2299 centry->cc_chain = NULL;
2300
2301 centry->cc_aging_dm &= ~(FOUND_IN_HASH_DM|FOUND_HOLD_OVER_DM);
2302
2303 /*
2304 * Do pagelist io mutual exclusion now if we did not do
2305 * it above.
2306 */
2307
2308 if (pageio && SET_CENTRY_PAGEIO(centry)) {
2309 /* flusher not done with pageio */
2310 sdbc_prefetch_pageio2++;
2311
2312 /*
2313 * prefetch: skip leading busy
2314 * or truncate at busy block
2315 */
2316 CLEAR_CENTRY_INUSE(centry);
2317 if (ioent == NULL)
2318 goto skip;
2319 sdbc_prefetch_busy_cnt++;
2320 fba_orig_len -= fba_len;
2321 fba_len = 0;
2322 centry = lentry; /* backup */
2323 break;
2324 }
2325
2326 pageio = 0;
2327
2328 fba_len -= st_cblk_len;
2329
2330 if (ioent == NULL) {
2331 if (!SDBC_VALID_BITS(st_cblk_off, st_cblk_len,
2332 centry)) {
2333 io_pos = BLK_TO_FBA_NUM(cblk) + st_cblk_off;
2334 ioent = last_ioent = centry;
2335 } else {
2336 DATA_LOG(SDF_ALLOC, centry, st_cblk_off,
2337 st_cblk_len);
2338 DTRACE_PROBE4(_sd_prefetch_buf_data1,
2339 uint64_t, (uint64_t)(BLK_TO_FBA_NUM(cblk) +
2340 st_cblk_off), int, st_cblk_len,
2341 char *, *(int64_t *)(centry->cc_data +
2342 FBA_SIZE(st_cblk_off)), char *,
2343 *(int64_t *)(centry->cc_data +
2344 FBA_SIZE(st_cblk_off + st_cblk_len) - 8));
2345 }
2346
2347 handle->bh_centry = centry;
2348 st_cblk_off = 0;
2349 st_cblk_len = (sdbc_cblk_fba_t)
2350 ((fba_len > (nsc_size_t)BLK_FBAS) ?
2351 BLK_FBAS : fba_len);
2352 } else {
2353 if (!SDBC_VALID_BITS(st_cblk_off, st_cblk_len, centry))
2354 last_ioent = centry;
2355 else {
2356 DTRACE_PROBE4(_sd_prefetch_buf_data2,
2357 uint64_t, (uint64_t)(BLK_TO_FBA_NUM(cblk) +
2358 st_cblk_off), int, st_cblk_len,
2359 char *, *(int64_t *)(centry->cc_data +
2360 FBA_SIZE(st_cblk_off)), char *,
2361 *(int64_t *)(centry->cc_data +
2362 FBA_SIZE(st_cblk_off + st_cblk_len) - 8));
2363 }
2364
2365 lentry->cc_chain = centry;
2366 if (fba_len < (nsc_size_t)BLK_FBAS)
2367 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
2368 }
2369 lentry = centry;
2370 cblk++;
2371
2372 /* if this block has a new identity clear prefetch history */
2373 if (this_entry_type != HASH_ENTRY_DM)
2374 centry->cc_aging_dm &=
2375 ~(PREFETCH_BUF_I | PREFETCH_BUF_E);
2376
2377 centry->cc_aging_dm &= ~(ENTRY_FIELD_DM);
2378 centry->cc_aging_dm |= this_entry_type | PREFETCH_BUF_E;
2379 if (flag & NSC_METADATA)
2380 centry->cc_aging_dm |= STICKY_METADATA_DM;
2381
2382 --request_blocks;
2383 } while (fba_len > 0);
2384
2385
2386 if (locked) {
2387 rw_exit(&sdbc_queue_lock);
2388 locked = 0;
2389 }
2390
2391 sdbc_centry_alloc_end(&alloc_tok);
2392
2393 if (centry) {
2394 centry->cc_chain = NULL;
2395 if (sts = _sd_setup_category_on_type(handle->bh_centry)) {
2396 (void) _sd_free_buf(handle);
2397 goto done;
2398 }
2399
2400 (void) _sd_setup_mem_chaining(handle->bh_centry, 0);
2401 }
2402
2403
2404 if (ioent) {
2405 /* prefetch: trailing valid can be released, adjust len */
2406 if ((centry != last_ioent)) {
2407 centry = last_ioent->cc_chain;
2408 last_ioent->cc_chain = NULL;
2409 while (centry) {
2410 lentry = centry->cc_chain;
2411 centry->cc_aging_dm &= ~PREFETCH_BUF_E;
2412 _sd_centry_release(centry);
2413 centry = lentry;
2414 sdbc_prefetch_trailing++;
2415 }
2416 fba_len = (CENTRY_BLK(last_ioent) -
2417 CENTRY_BLK(ioent) + 1) * BLK_FBAS -
2418 BLK_FBA_OFF(io_pos);
2419 fba_orig_len = fba_len + (io_pos - fba_pos);
2420 }
2421
2422 _SD_DISCONNECT_CALLBACK(handle);
2423 sts = _sd_doread(handle, ioent, io_pos,
2424 (fba_pos + fba_orig_len - io_pos), flag);
2425 if (sts > 0)
2426 (void) _sd_free_buf(handle);
2427 } else {
2428 CACHE_FBA_READ(cd, fba_orig_len);
2429 CACHE_READ_HIT;
2430 FBA_READ_IO_KSTATS(cd, FBA_SIZE(fba_orig_len));
2431
2432 sts = NSC_HIT;
2433 }
2434 done:
2435 if (locked)
2436 rw_exit(&sdbc_queue_lock);
2437
2438 return (sts);
2439 }
2440
2441
2442 /*
2443 * _sd_cc_wait - wait for inuse cache block to become available
2444 * Usage:
2445 * if (SET_CENTRY_INUSE(centry)) {
2446 * _sd_cc_wait(cd, blk, centry, CC_INUSE);
2447 * goto try_again;
2448 * }
2449 * -or-
2450 * if (SET_CENTRY_PAGEIO(centry)) {
2451 * _sd_cc_wait(cd, blk, centry, CC_PAGEIO);
2452 * goto try_again;
2453 * }
2454 */
2455 void
2456 _sd_cc_wait(int cd, nsc_off_t cblk, _sd_cctl_t *centry, int flag)
2457 {
2458 volatile ushort_t *waiters;
2459 volatile uchar_t *uflag;
2460
2461 if (flag == CC_INUSE) {
2462 waiters = &(centry->cc_await_use);
2463 uflag = &(CENTRY_INUSE(centry));
2464 } else if (flag == CC_PAGEIO) {
2465 waiters = &(centry->cc_await_page);
2466 uflag = &(CENTRY_PAGEIO(centry));
2467 } else {
2468 /* Oops! */
2469 #ifdef DEBUG
2470 cmn_err(CE_WARN, "!_sd_cc_wait: unknown flag value (%x)", flag);
2471 #endif
2472 return;
2473 }
2474
2475 mutex_enter(¢ry->cc_lock);
2476 if (CC_CD_BLK_MATCH(cd, cblk, centry) && (*uflag) != 0) {
2477 (*waiters)++;
2478 sd_serialize();
2479 if ((*uflag) != 0) {
2480 unsigned stime = nsc_usec();
2481 cv_wait(¢ry->cc_blkcv, ¢ry->cc_lock);
2482 (*waiters)--;
2483 mutex_exit(¢ry->cc_lock);
2484 SDTRACE(ST_INFO|SDF_ENT_GET,
2485 cd, 0, BLK_TO_FBA_NUM(cblk), (nsc_usec()-stime), 0);
2486 } else {
2487 (*waiters)--;
2488 mutex_exit(¢ry->cc_lock);
2489 }
2490 } else
2491 mutex_exit(¢ry->cc_lock);
2492
2493 }
2494
2495 /*
2496 * _sd_alloc_buf - Allocate a vector of buffers for io.
2497 *
2498 * ARGUMENTS:
2499 * cd - Cache descriptor (from a previous open)
2500 * fba_pos - disk position (512-byte FBAs)
2501 * fba_len - length in disk FBAs.
2502 * flag - allocation type. Flag is one or more of
2503 * NSC_RDBUF, NSC_WRBUF, NSC_NOBLOCK and hints.
2504 * NSC_RDAHEAD - prefetch for future read.
2505 * handle_p - pointer to a handle pointer.
2506 * If the handle pointer is non-null, its used as a
2507 * pre-allocated handle. Else a new handle will be allocated
2508 * and stored in *handle_p
2509 *
2510 * RETURNS:
2511 * errno if return > 0.
2512 * else NSC_HIT or NSC_DONE on success
2513 * or NSC_PENDING on io in progress and NSC_NOBLOCK
2514 * specified in the flag.
2515 * USAGE:
2516 * This routine allocates the cache blocks requested and creates a list
2517 * of entries for this request.
2518 * If NSC_NOBLOCK was not specified, this call could block on read io.
2519 * If flag specified NSC_RDBUF and the request is not an entire
2520 * hit, an io is initiated.
2521 */
2522 int
2523 _sd_alloc_buf(blind_t xcd, nsc_off_t fba_pos, nsc_size_t fba_len, int flag,
2524 _sd_buf_handle_t **handle_p)
2525 {
2526 int cd = (int)(unsigned long)xcd;
2527 _sd_cd_info_t *cdi;
2528 _sd_buf_handle_t *handle;
2529 int sts;
2530 nsc_off_t st_cblk, cblk; /* position of start and temp cache block */
2531 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
2532 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
2533 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
2534 nsc_off_t io_pos; /* offset in FBA's */
2535 _sd_bufvec_t *bufvec;
2536 _sd_cctl_t *centry, *lentry, *ioent = NULL;
2537 nsc_size_t fba_orig_len = fba_len; /* FBA length of orig request */
2538 int stall, pageio;
2539 unsigned char cc_flag;
2540 int this_entry_type;
2541 int locked = 0;
2542 nsc_size_t dmchain_request_blocks; /* size of dmchain in cache blocks */
2543 sdbc_allocbuf_t alloc_tok = {0};
2544 int min_frag = 0; /* frag statistics */
2545 int max_frag = 0; /* frag statistics */
2546 int nfrags = 0; /* frag statistics */
2547 #ifdef DEBUG
2548 int err = 0;
2549 #endif
2550
2551
2552 ASSERT(*handle_p != NULL);
2553 handle = *handle_p;
2554
2555 if (_sdbc_shutdown_in_progress)
2556 return (EIO);
2557
2558 if (xcd == NSC_ANON_CD)
2559 cd = _CD_NOHASH;
2560
2561 KSTAT_RUNQ_ENTER(cd);
2562
2563 /*
2564 * Force large writes on nvram systems to be write-through to
2565 * avoid the (slow) bcopy into nvram.
2566 */
2567
2568 if (flag & NSC_WRBUF) {
2569 if (fba_len > (nsc_size_t)sdbc_wrthru_len) {
2570 flag |= NSC_WRTHRU;
2571 }
2572 }
2573
2574 #ifdef DEBUG
2575 if (sdbc_pageio_debug != SDBC_PAGEIO_OFF) {
2576 switch (sdbc_pageio_debug) {
2577 case SDBC_PAGEIO_RDEV:
2578 if (cd != _CD_NOHASH &&
2579 sdbc_pageio_rdev != (dev_t)-1 &&
2580 _sd_cache_files[cd].cd_crdev == sdbc_pageio_rdev)
2581 flag |= NSC_PAGEIO;
2582 break;
2583
2584 case SDBC_PAGEIO_RAND:
2585 if ((nsc_lbolt() % 3) == 0)
2586 flag |= NSC_PAGEIO;
2587 break;
2588
2589 case SDBC_PAGEIO_ALL:
2590 flag |= NSC_PAGEIO;
2591 break;
2592 }
2593 }
2594 #endif /* DEBUG */
2595
2596 if (fba_len > (nsc_size_t)BLK_FBAS) {
2597 rw_enter(&sdbc_queue_lock, RW_WRITER);
2598 locked = 1;
2599 }
2600
2601 /*
2602 * _CD_NOHASH: client wants temporary (not hashed) cache memory
2603 * not associated with a local disk. Skip local disk checks.
2604 */
2605 if (cd == _CD_NOHASH) {
2606 flag &= ~(NSC_RDBUF | NSC_WRBUF | NSC_RDAHEAD);
2607 handle = *handle_p;
2608 handle->bh_flag |= NSC_HACTIVE;
2609 goto setup;
2610 }
2611
2612 SDTRACE(ST_ENTER|SDF_ALLOCBUF, cd, fba_len, fba_pos, flag, 0);
2613
2614
2615 if ((flag & NSC_RDAHEAD) && _sd_prefetch_opt) {
2616 sts = _sd_prefetch_buf(cd, fba_pos, fba_len, flag, handle,
2617 locked);
2618 goto done;
2619 }
2620
2621 #if !defined(_SD_NOCHECKS)
2622 if (flag & NSC_RDAHEAD) { /* _sd_prefetch_opt == 0 */
2623 nsc_size_t file_size; /* file_size in FBA's */
2624 /* prefetch: truncate if req'd */
2625 if (fba_len > sdbc_max_fbas)
2626 fba_len = sdbc_max_fbas;
2627 file_size = _sd_cache_files[(cd)].cd_info->sh_filesize;
2628 if ((fba_pos + fba_len) > file_size) {
2629 fba_len = file_size - fba_pos;
2630 #ifdef NSC_MULTI_TERABYTE
2631 if ((int64_t)fba_len <= 0) {
2632 #else
2633 if ((int32_t)fba_len <= 0) {
2634 #endif
2635 sts = EIO;
2636 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_len,
2637 fba_pos, flag, sts);
2638 goto done;
2639 }
2640 }
2641 } else
2642 if (sts = _sd_check_buffer_alloc(cd, fba_pos, fba_len, handle_p)) {
2643 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_len, fba_pos, flag, sts);
2644 goto done;
2645 }
2646 #endif
2647 if (fba_len == 0) {
2648 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_len, fba_pos,
2649 flag, EINVAL);
2650 sts = EINVAL;
2651 goto done;
2652 }
2653
2654 handle->bh_flag |= NSC_HACTIVE;
2655 cdi = &_sd_cache_files[cd];
2656
2657 if (cdi->cd_recovering) {
2658 /*
2659 * If recovering this device, then block all allocates
2660 * for reading or writing. If we allow reads then
2661 * this path could see old data before we recover.
2662 * If we allow writes then new data could be overwritten
2663 * by old data.
2664 * This is clearly still not a complete solution as
2665 * the thread doing this allocate could conceivably be
2666 * by this point (and in _sd_write/_sd_read for that matter
2667 * which don't even have this protection). But this type
2668 * of path seems to only exist in a failover situation
2669 * where a device has failed on the other node and works
2670 * on this node so the problem is not a huge one but exists
2671 * never the less.
2672 */
2673 if (sts = _sd_recovery_wblk_wait(cd)) {
2674 handle->bh_flag &= ~NSC_HACTIVE;
2675 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_len, fba_pos,
2676 flag, sts);
2677 goto done;
2678 }
2679 }
2680
2681 /* write & disk failed, return error immediately */
2682 if ((flag & NSC_WRBUF) && cdi->cd_info->sh_failed) {
2683 handle->bh_flag &= ~NSC_HACTIVE;
2684 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_len, fba_pos, flag, EIO);
2685 sts = EIO;
2686 goto done;
2687 }
2688
2689 setup:
2690
2691 _SD_SETUP_HANDLE(handle, cd, fba_pos, fba_len, flag);
2692 handle->bh_centry = NULL;
2693 bufvec = handle->bh_bufvec;
2694 if (flag & NSC_RDAHEAD) { /* _sd_prefetch_opt == 0 */
2695 /* CKD prefetch: bufvec not req'd, use placeholder */
2696 bufvec->bufaddr = NULL;
2697 bufvec->bufvmeaddr = NULL;
2698 bufvec->buflen = 0;
2699 bufvec = _prefetch_sb_vec;
2700 }
2701 st_cblk = FBA_TO_BLK_NUM(fba_pos);
2702 st_cblk_off = BLK_FBA_OFF(fba_pos);
2703 st_cblk_len = BLK_FBAS - st_cblk_off;
2704 if ((nsc_size_t)st_cblk_len >= fba_len) {
2705 end_cblk_len = 0;
2706 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
2707 } else
2708 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
2709 cblk = st_cblk;
2710
2711
2712 /*
2713 * count number of blocks on chain that is required
2714 */
2715
2716 /* middle piece */
2717 dmchain_request_blocks =
2718 (fba_len - (st_cblk_len + end_cblk_len)) >> BLK_FBA_SHFT;
2719
2720 /* start piece */
2721 ++dmchain_request_blocks;
2722
2723 /* end piece */
2724 if (end_cblk_len)
2725 ++dmchain_request_blocks;
2726
2727
2728 cc_flag = 0;
2729 if ((handle->bh_flag & NSC_PINNABLE) && (handle->bh_flag & NSC_WRBUF))
2730 cc_flag |= CC_PINNABLE;
2731 if (handle->bh_flag & (NSC_NOCACHE|NSC_SEQ_IO))
2732 cc_flag |= CC_QHEAD;
2733 lentry = NULL;
2734 stall = 0;
2735
2736 do {
2737 pageio = ((flag & NSC_PAGEIO) != 0 || sdbc_pageio_always != 0);
2738 cget:
2739 if ((centry = (_sd_cctl_t *)
2740 _sd_hash_search(cd, cblk, _sd_htable)) != 0) {
2741
2742 if (SET_CENTRY_INUSE(centry)) {
2743 /* already inuse: wait for block, retry */
2744 sdbc_allocb_inuse++;
2745 if (locked)
2746 rw_exit(&sdbc_queue_lock);
2747 _sd_cc_wait(cd, cblk, centry, CC_INUSE);
2748 if (locked)
2749 rw_enter(&sdbc_queue_lock, RW_WRITER);
2750 goto cget;
2751 }
2752
2753 /*
2754 * bug 4529671
2755 * now that we own the centry make sure that
2756 * it is still good. it could have been processed
2757 * by _sd_dealloc_dm() in the window between
2758 * _sd_hash_search() and SET_CENTRY_INUSE().
2759 */
2760 if ((_sd_cctl_t *)
2761 _sd_hash_search(cd, cblk, _sd_htable) != centry) {
2762 sdbc_allocb_deallocd++;
2763 #ifdef DEBUG
2764 cmn_err(CE_WARN,
2765 "!centry %p cd %d cblk %" NSC_SZFMT
2766 " fba_len %" NSC_SZFMT " lost to dealloc?! "
2767 "cc_data %p", (void *)centry, cd, cblk,
2768 fba_orig_len, (void *)centry->cc_data);
2769 #endif
2770
2771 CLEAR_CENTRY_INUSE(centry);
2772 goto cget;
2773 }
2774
2775 if (CC_CD_BLK_MATCH(cd, cblk, centry)) {
2776 /*
2777 * Do pagelist io mutual exclusion
2778 * before messing with the centry.
2779 */
2780 if (pageio && SET_CENTRY_PAGEIO(centry)) {
2781 /* wait for flusher to finish pageio */
2782 sdbc_allocb_pageio1++;
2783
2784 CLEAR_CENTRY_INUSE(centry);
2785 if (locked)
2786 rw_exit(&sdbc_queue_lock);
2787 _sd_cc_wait(cd, cblk, centry,
2788 CC_PAGEIO);
2789 if (locked)
2790 rw_enter(&sdbc_queue_lock,
2791 RW_WRITER);
2792 goto cget;
2793 }
2794
2795 sdbc_allocb_hit++;
2796 this_entry_type = HASH_ENTRY_DM;
2797 pageio = 0;
2798 centry->cc_toflush = 0;
2799
2800 centry->cc_hits++;
2801
2802 /* this will reset the age flag */
2803 sdbc_centry_init_dm(centry);
2804
2805 DTRACE_PROBE1(_sd_alloc_buf1,
2806 _sd_cctl_t *, centry);
2807 } else {
2808 /* block mismatch: release, alloc new block */
2809 sdbc_allocb_lost++;
2810
2811 CLEAR_CENTRY_INUSE(centry);
2812
2813 goto cget;
2814
2815 }
2816 } else {
2817 centry = sdbc_centry_alloc(cd, cblk,
2818 dmchain_request_blocks, &stall,
2819 &alloc_tok, locked ? ALLOC_LOCKED : 0);
2820
2821 /*
2822 * dmchaining adjustment.
2823 * if centry was obtained from the dmchain
2824 * then clear local pageio variable because the
2825 * centry already has cc_pageio set.
2826 */
2827 if (CENTRY_PAGEIO(centry))
2828 pageio = 0;
2829
2830 DTRACE_PROBE1(_sd_alloc_buf2, _sd_cctl_t *, centry);
2831
2832 this_entry_type = ELIGIBLE_ENTRY_DM;
2833 if (centry->cc_aging_dm & FOUND_IN_HASH_DM)
2834 this_entry_type = HASH_ENTRY_DM;
2835 else {
2836 if (centry->cc_aging_dm & FOUND_HOLD_OVER_DM)
2837 this_entry_type = HOLD_ENTRY_DM;
2838 }
2839 }
2840
2841 centry->cc_aging_dm &= ~(FOUND_IN_HASH_DM|FOUND_HOLD_OVER_DM);
2842
2843 /*
2844 * Do pagelist io mutual exclusion now if we did not do
2845 * it above.
2846 */
2847
2848 if (pageio && SET_CENTRY_PAGEIO(centry)) {
2849 /* wait for flusher to finish pageio */
2850 sdbc_allocb_pageio2++;
2851
2852
2853 CLEAR_CENTRY_INUSE(centry);
2854 if (locked)
2855 rw_exit(&sdbc_queue_lock);
2856 _sd_cc_wait(cd, cblk, centry, CC_PAGEIO);
2857 if (locked)
2858 rw_enter(&sdbc_queue_lock, RW_WRITER);
2859 goto cget;
2860 }
2861
2862 pageio = 0;
2863
2864 if (CENTRY_DIRTY(centry)) {
2865 /*
2866 * end action might set PEND_DIRTY flag
2867 * must lock if need to change flag bits
2868 */
2869 if (centry->cc_flag != (centry->cc_flag | cc_flag)) {
2870 /* was FAST */
2871 mutex_enter(¢ry->cc_lock);
2872 centry->cc_flag |= cc_flag;
2873 /* was FAST */
2874 mutex_exit(¢ry->cc_lock);
2875 }
2876 } else
2877 centry->cc_flag |= cc_flag;
2878
2879 centry->cc_chain = NULL;
2880
2881 /*
2882 * step 0:check valid bits in each cache ele as
2883 * the chain grows - set ioent/io_pos to first
2884 * instance of invalid data
2885 */
2886 if (cblk == st_cblk) {
2887 handle->bh_centry = centry;
2888 fba_len -= st_cblk_len;
2889 lentry = centry;
2890 if (flag & NSC_RDBUF) {
2891 if (!SDBC_VALID_BITS(st_cblk_off, st_cblk_len,
2892 centry)) {
2893 io_pos = fba_pos;
2894 ioent = centry;
2895 } else {
2896 DATA_LOG(SDF_ALLOC, centry, st_cblk_off,
2897 st_cblk_len);
2898
2899 DTRACE_PROBE4(_sd_alloc_data1,
2900 uint64_t, (uint64_t)
2901 (BLK_TO_FBA_NUM(cblk) +
2902 st_cblk_off), int, st_cblk_len,
2903 char *, *(int64_t *)
2904 (centry->cc_data +
2905 FBA_SIZE(st_cblk_off)),
2906 char *, *(int64_t *)
2907 (centry->cc_data +
2908 FBA_SIZE(st_cblk_off + st_cblk_len)
2909 - 8));
2910 }
2911 }
2912 cblk++;
2913 } else if (fba_len == (nsc_size_t)end_cblk_len) {
2914 lentry->cc_chain = centry;
2915 fba_len -= end_cblk_len;
2916 if (flag & NSC_RDBUF) {
2917 if (ioent == NULL) {
2918 if (!SDBC_VALID_BITS(0, end_cblk_len,
2919 centry)) {
2920 io_pos = BLK_TO_FBA_NUM(cblk);
2921 ioent = centry;
2922 } else {
2923 DATA_LOG(SDF_ALLOC, centry, 0,
2924 end_cblk_len);
2925
2926 DTRACE_PROBE4(_sd_alloc_data2,
2927 uint64_t,
2928 BLK_TO_FBA_NUM(cblk),
2929 int, end_cblk_len,
2930 char *, *(int64_t *)
2931 (centry->cc_data),
2932 char *, *(int64_t *)
2933 (centry->cc_data +
2934 FBA_SIZE(end_cblk_len)
2935 - 8));
2936 }
2937 }
2938 }
2939 } else {
2940 lentry->cc_chain = centry;
2941 lentry = centry;
2942 fba_len -= BLK_FBAS;
2943 if (flag & NSC_RDBUF) {
2944 if (ioent == NULL) {
2945 if (!FULLY_VALID(centry)) {
2946 io_pos = BLK_TO_FBA_NUM(cblk);
2947 ioent = centry;
2948 } else {
2949 DATA_LOG(SDF_ALLOC, centry, 0,
2950 BLK_FBAS);
2951
2952 DTRACE_PROBE4(_sd_alloc_data3,
2953 uint64_t, (uint64_t)
2954 BLK_TO_FBA_NUM(cblk),
2955 int, BLK_FBAS,
2956 char *, *(int64_t *)
2957 (centry->cc_data),
2958 char *, *(int64_t *)
2959 (centry->cc_data +
2960 FBA_SIZE(BLK_FBAS) - 8));
2961 }
2962 }
2963 }
2964 cblk++;
2965 }
2966
2967 /* if this block has a new identity clear prefetch history */
2968 if (this_entry_type != HASH_ENTRY_DM)
2969 centry->cc_aging_dm &=
2970 ~(PREFETCH_BUF_I | PREFETCH_BUF_E);
2971
2972 centry->cc_aging_dm &= ~(ENTRY_FIELD_DM);
2973 centry->cc_aging_dm |= this_entry_type;
2974 if (flag & NSC_METADATA)
2975 centry->cc_aging_dm |= STICKY_METADATA_DM;
2976
2977 --dmchain_request_blocks;
2978 } while (fba_len);
2979
2980 if (locked) {
2981 rw_exit(&sdbc_queue_lock);
2982 locked = 0;
2983 }
2984
2985 ASSERT(dmchain_request_blocks == 0);
2986
2987 /*
2988 * do any necessary cleanup now that all the blocks are allocated.
2989 */
2990 sdbc_centry_alloc_end(&alloc_tok);
2991
2992 /* be sure you nul term. the chain */
2993 centry->cc_chain = NULL;
2994
2995 /*
2996 * step one: establish HOST/PARASITE/OTHER relationships
2997 * between the centry ele in the list and calc the alloc size
2998 * (fill in CATAGORY based on TYPE and immediate neighbors)
2999 */
3000 if (sts = _sd_setup_category_on_type(handle->bh_centry)) {
3001 #ifdef DEBUG
3002 err = _sd_free_buf(handle);
3003 if (err) {
3004 cmn_err(CE_WARN, "!sdbc(_sd_alloc_buf): _sd_free_buf "
3005 "failed: err:%d handle:%p", err, (void *)handle);
3006 }
3007 #else
3008 (void) _sd_free_buf(handle);
3009 #endif
3010 goto done;
3011 }
3012
3013 /*
3014 * step two: alloc the needed mem and fill in the data and chaining
3015 * fields (leave bufvec for step three)
3016 */
3017 (void) _sd_setup_mem_chaining(handle->bh_centry, 0);
3018
3019 /*
3020 * step three: do the bufvec
3021 */
3022 fba_len = fba_orig_len;
3023 centry = handle->bh_centry;
3024 bufvec = handle->bh_bufvec;
3025
3026 while (centry) {
3027 DTRACE_PROBE3(_sd_alloc_buf_centrys, _sd_cctl_t *, centry,
3028 int, cd, uint64_t,
3029 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(centry)));
3030
3031 if (fba_len == fba_orig_len) {
3032 bufvec->bufaddr = (centry->cc_data +
3033 FBA_SIZE(st_cblk_off));
3034 bufvec->bufvmeaddr = 0; /* not used */
3035 bufvec->buflen = FBA_SIZE(st_cblk_len);
3036 bufvec++;
3037 fba_len -= st_cblk_len;
3038 } else if (fba_len == (nsc_size_t)end_cblk_len) {
3039 _sd_bufvec_t *pbufvec = bufvec - 1;
3040
3041 if ((pbufvec->bufaddr + pbufvec->buflen) ==
3042 centry->cc_data) {
3043 /* contiguous */
3044 pbufvec->buflen += FBA_SIZE(end_cblk_len);
3045 } else {
3046
3047 bufvec->bufaddr = centry->cc_data;
3048 bufvec->bufvmeaddr = 0; /* not used */
3049 bufvec->buflen = FBA_SIZE(end_cblk_len);
3050 bufvec++;
3051 }
3052
3053 fba_len -= end_cblk_len;
3054 } else {
3055 _sd_bufvec_t *pbufvec = bufvec - 1;
3056
3057 if ((pbufvec->bufaddr + pbufvec->buflen) ==
3058 centry->cc_data) {
3059 /* contiguous */
3060 pbufvec->buflen += CACHE_BLOCK_SIZE;
3061 } else {
3062
3063 bufvec->bufaddr = centry->cc_data;
3064 bufvec->bufvmeaddr = 0; /* not used */
3065 bufvec->buflen = CACHE_BLOCK_SIZE;
3066 bufvec++;
3067 }
3068
3069 fba_len -= BLK_FBAS;
3070 }
3071
3072 centry = centry->cc_chain;
3073 }
3074
3075 /* be sure you nul term. the chain */
3076 bufvec->bufaddr = NULL;
3077 bufvec->bufvmeaddr = 0;
3078 bufvec->buflen = 0;
3079
3080 /* frag statistics */
3081 {
3082 _sd_bufvec_t *tbufvec;
3083
3084 for (tbufvec = handle->bh_bufvec; tbufvec != bufvec;
3085 ++tbufvec) {
3086 if ((min_frag > tbufvec->buflen) || (min_frag == 0))
3087 min_frag = tbufvec->buflen;
3088
3089 if (max_frag < tbufvec->buflen)
3090 max_frag = tbufvec->buflen;
3091 }
3092
3093 nfrags = bufvec - handle->bh_bufvec;
3094 min_frag = FBA_LEN(min_frag);
3095 max_frag = FBA_LEN(max_frag);
3096 }
3097
3098 /* buffer memory frag stats */
3099 DTRACE_PROBE4(_sd_alloc_buf_frag, uint64_t, (uint64_t)fba_orig_len,
3100 int, nfrags, int, min_frag, int, max_frag);
3101
3102
3103 if (flag & NSC_WRBUF) {
3104 if (_SD_IS_WRTHRU(handle))
3105 goto alloc_done;
3106 if (_sd_alloc_write(handle->bh_centry, &stall)) {
3107 _sd_unblock(&_sd_flush_cv);
3108 handle->bh_flag |= NSC_FORCED_WRTHRU;
3109 } else {
3110 for (centry = handle->bh_centry;
3111 centry; centry = centry->cc_chain) {
3112
3113 CENTRY_SET_FTPOS(centry);
3114 SSOP_SETCENTRY(sdbc_safestore,
3115 centry->cc_write);
3116 }
3117 }
3118 }
3119
3120 alloc_done:
3121 if (locked) {
3122 rw_exit(&sdbc_queue_lock);
3123 locked = 0;
3124 }
3125 if (ioent) {
3126 _SD_DISCONNECT_CALLBACK(handle);
3127 sts = _sd_doread(handle, ioent, io_pos,
3128 (fba_pos + fba_orig_len - io_pos), flag);
3129 if (sts > 0)
3130 (void) _sd_free_buf(handle);
3131 } else
3132 if (flag & NSC_RDBUF) {
3133 CACHE_FBA_READ(cd, fba_orig_len);
3134 CACHE_READ_HIT;
3135 FBA_READ_IO_KSTATS(cd, FBA_SIZE(fba_orig_len));
3136
3137 sts = NSC_HIT;
3138 } else
3139 sts = (stall) ? NSC_DONE : NSC_HIT;
3140
3141 SDTRACE(ST_EXIT|SDF_ALLOCBUF, cd, fba_orig_len, fba_pos, flag, sts);
3142
3143 done:
3144 if (locked)
3145 rw_exit(&sdbc_queue_lock);
3146
3147 KSTAT_RUNQ_EXIT(cd);
3148
3149 return (sts);
3150 }
3151
3152 /*
3153 * consistency checking for ccents
3154 */
3155
3156 #define ELIGIBLE(p) (p & ELIGIBLE_ENTRY_DM)
3157 #define HOLD(p) (p & HOLD_ENTRY_DM)
3158 #define HASHE(p) (p & HASH_ENTRY_DM)
3159
3160 #define HOST(p) (p & HOST_ENTRY_DM)
3161 #define PARA(p) (p & PARASITIC_ENTRY_DM)
3162 #define OTHER(p) \
3163 (!(p & (HOST_ENTRY_DM | PARASITIC_ENTRY_DM | ELIGIBLE_ENTRY_DM)))
3164
3165 #define AVAIL(p) (p & AVAIL_ENTRY_DM)
3166
3167 /*
3168 * sdbc_check_cctl_cot -- consistency check for _sd_setup_category_on_type()
3169 * may only be called on entry to state machine (when ccent is either
3170 * ELIGIBLE_ENTRY_DM, HOLD_ENTRY_DM or HASH_ENTRY_DM).
3171 *
3172 * print message or panic (DEBUG) if inconsistency detected.
3173 */
3174 static int
3175 sdbc_check_cctl_cot(_sd_cctl_t *centry)
3176 {
3177 uint_t age;
3178 int size;
3179 uchar_t *data;
3180 int host_or_other;
3181 int para;
3182 int ccent_ok = 1;
3183
3184 age = centry->cc_aging_dm;
3185 size = centry->cc_alloc_size_dm;
3186 data = centry->cc_data;
3187 host_or_other = size && data;
3188 para = !size && data;
3189
3190 /*
3191 * on entry to _sd_setup_category_on_type(),
3192 * one of three mutually exclusive entry field bits must be set
3193 */
3194
3195 switch ((age & (ELIGIBLE_ENTRY_DM | HOLD_ENTRY_DM | HASH_ENTRY_DM))) {
3196 case ELIGIBLE_ENTRY_DM:
3197 case HOLD_ENTRY_DM:
3198 case HASH_ENTRY_DM:
3199 /* ok */
3200 break;
3201 default:
3202 /* zero or multiple flag bits */
3203 ccent_ok = 0;
3204 break;
3205 }
3206
3207 /* categories are mutually exclusive */
3208 if (HOST(age) && PARA(age))
3209 ccent_ok = 0;
3210
3211 /* these bits should be cleared out (STICKY_METADATA_DM not used) */
3212 if (age & (AVAIL_ENTRY_DM | FOUND_HOLD_OVER_DM | FOUND_IN_HASH_DM |
3213 STICKY_METADATA_DM))
3214 ccent_ok = 0;
3215
3216 /* eligible has no data and no size */
3217 if (ELIGIBLE(age) && (size || data))
3218 ccent_ok = 0;
3219
3220 /* parasite has zero size and non-zero data */
3221 if (PARA(age) && !para)
3222 ccent_ok = 0;
3223
3224 /* host has non-zero size and non-zero data */
3225 if (HOST(age) && !host_or_other)
3226 ccent_ok = 0;
3227
3228 /* "other" is just like a host */
3229 if (OTHER(age) && !host_or_other)
3230 ccent_ok = 0;
3231
3232 /* a HOLD or a HASH must have a size */
3233 if ((size) && !(age & (HASH_ENTRY_DM | HOLD_ENTRY_DM)))
3234 ccent_ok = 0;
3235
3236 if (!ccent_ok)
3237 cmn_err(cmn_level,
3238 "!sdbc(sdbc_check_cctl_cot): inconsistent ccent %p "
3239 "age %x size %d data %p", (void *)centry, age, size,
3240 (void *)data);
3241
3242 return (ccent_ok);
3243 }
3244
3245 /*
3246 * sdbc_mark_cctl_cot -- mark cctls bad and invalidate when
3247 * inconsistency found in _sd_setup_category_on_type()
3248 * returns nothing
3249 *
3250 * Note: this is an error recovery path that is triggered when an
3251 * inconsistency in a cctl is detected. _sd_centry_release() will take
3252 * these cache entries out of circulation and place them on a separate list
3253 * for debugging purposes.
3254 */
3255 void
3256 sdbc_mark_cctl_cot(_sd_cctl_t *header, _sd_cctl_t *centry)
3257 {
3258 _sd_cctl_t *cur_ent = header;
3259
3260 /* the entire chain is guilty by association */
3261 while (cur_ent) {
3262
3263 (void) _sd_hash_delete((struct _sd_hash_hd *)cur_ent,
3264 _sd_htable);
3265
3266 cur_ent->cc_aging_dm |= BAD_CHAIN_DM;
3267
3268 cur_ent = cur_ent->cc_chain;
3269 }
3270
3271 centry->cc_aging_dm |= BAD_ENTRY_DM; /* this is the problem child */
3272 }
3273
3274 /*
3275 * _sd_setup_category_on_type(_sd_cctl_t *) - Setup the centry CATEGORY based on
3276 * centry TYPE and immediate neighbors. Identify each eligible (ie not HASH)
3277 * centry as a host/parasite. host actually have memory allocated to
3278 * them and parasites are chained to the host and point to page offsets within
3279 * the host's memory.
3280 *
3281 * RETURNS:
3282 * 0 on success, EINTR if inconsistency detected in centry
3283 *
3284 * Note:
3285 * none
3286 */
3287 static int
3288 _sd_setup_category_on_type(_sd_cctl_t *header)
3289 {
3290 _sd_cctl_t *prev_ent, *next_ent, *centry;
3291 _sd_cctl_t *anchor = NULL;
3292 int current_pest_count, local_max_dyn_list;
3293 int cl;
3294 int ret = 0;
3295
3296 ASSERT(header);
3297
3298 if (sdbc_use_dmchain)
3299 local_max_dyn_list = max_dm_queues - 1;
3300 else {
3301 /* pickup a fresh copy - has the world changed */
3302 local_max_dyn_list = dynmem_processing_dm.max_dyn_list;
3303 }
3304
3305 prev_ent = 0;
3306 centry = header;
3307 next_ent = centry->cc_chain;
3308 current_pest_count = 0;
3309 cl = 2;
3310
3311 /* try to recover from bad cctl */
3312 if (sdbc_check_cot && !sdbc_check_cctl_cot(centry))
3313 ret = EINTR;
3314
3315 while (cl && (ret == 0)) {
3316 switch (cl) {
3317 case (1): /* chain to next/monitor for completion */
3318 prev_ent = centry;
3319 centry = next_ent;
3320 next_ent = 0;
3321 cl = 0;
3322 if (centry) {
3323
3324 if (sdbc_check_cot &&
3325 !sdbc_check_cctl_cot(centry)) {
3326 ret = EINTR;
3327 break;
3328 }
3329
3330 next_ent = centry->cc_chain;
3331 cl = 2;
3332 }
3333 break;
3334
3335 case (2): /* vector to appropriate routine */
3336 if (!(centry->cc_aging_dm & ELIGIBLE_ENTRY_DM))
3337 cl = 5;
3338 else if (prev_ent && (prev_ent->cc_aging_dm &
3339 ELIGIBLE_ENTRY_DM))
3340 cl = 15;
3341 else
3342 cl = 10;
3343 break;
3344
3345 case (5): /* process NON-ELIGIBLE entries */
3346 if (!(centry->cc_aging_dm &
3347 (HASH_ENTRY_DM|HOLD_ENTRY_DM))) {
3348 /* no catagory */
3349
3350 /* consistency check */
3351 if (centry->cc_alloc_size_dm ||
3352 centry->cc_data) {
3353 cmn_err(cmn_level,
3354 "!sdbc(setup_cot): "
3355 "OTHER with data/size %p",
3356 (void *)centry);
3357
3358 ret = EINTR;
3359 break;
3360 }
3361
3362 centry->cc_aging_dm &=
3363 ~CATAGORY_ENTRY_DM;
3364 centry->cc_alloc_size_dm = BLK_SIZE(1);
3365 DTRACE_PROBE1(_sd_setup_category,
3366 _sd_cctl_t *, centry);
3367 }
3368 cl = 1;
3369 break;
3370
3371 /*
3372 * no prev entry (ie top of list) or no prev
3373 * ELIGIBLE entry
3374 */
3375 case (10):
3376 /*
3377 * this is an eligible entry, does it start
3378 * a list or is it a loner
3379 */
3380 /* consistency check */
3381 if (centry->cc_alloc_size_dm ||
3382 centry->cc_data) {
3383 cmn_err(cmn_level, "!sdbc(setup_cot): "
3384 "HOST with data/size %p",
3385 (void *)centry);
3386 ret = EINTR;
3387 break;
3388 }
3389
3390 if (next_ent && (next_ent->cc_aging_dm &
3391 ELIGIBLE_ENTRY_DM)) {
3392
3393
3394 /* it starts a list */
3395 /* host catagory */
3396 centry->cc_aging_dm |= HOST_ENTRY_DM;
3397 /* start out with one page */
3398 centry->cc_alloc_size_dm = BLK_SIZE(1);
3399 anchor = centry;
3400 DTRACE_PROBE1(_sd_setup_category,
3401 _sd_cctl_t *, anchor);
3402 cl = 1;
3403 } else {
3404 /*
3405 * it's a loner
3406 * drop status to no category and
3407 * restart
3408 */
3409 cl = 2;
3410 centry->cc_aging_dm &=
3411 ~ELIGIBLE_ENTRY_DM;
3412 }
3413 break;
3414
3415 case (15): /* default to parasite catagory */
3416
3417 /* consistency check */
3418 if (centry->cc_alloc_size_dm ||
3419 centry->cc_data) {
3420 cmn_err(cmn_level, "!sdbc(setup_cot): "
3421 "PARA with data/size %p",
3422 (void *)centry);
3423
3424 ret = EINTR;
3425 break;
3426 }
3427
3428 if (current_pest_count < local_max_dyn_list-1) {
3429 /* continue to grow the pest list */
3430 current_pest_count++;
3431 centry->cc_aging_dm |=
3432 PARASITIC_ENTRY_DM;
3433
3434 /*
3435 * offset of host ent mem this will pt
3436 * to
3437 */
3438 centry->cc_alloc_size_dm =
3439 anchor->cc_alloc_size_dm;
3440 /*
3441 * up the host mem req by one for
3442 * this parasite
3443 */
3444 DTRACE_PROBE1(_sd_setup_category,
3445 _sd_cctl_t *, centry);
3446
3447 anchor->cc_alloc_size_dm += BLK_SIZE(1);
3448
3449 cl = 1;
3450 } else {
3451 /*
3452 * term this pest list - restart fresh
3453 * on this entry
3454 */
3455 current_pest_count = 0;
3456 prev_ent->cc_aging_dm &=
3457 ~(HOST_ENTRY_DM|ELIGIBLE_ENTRY_DM);
3458 cl = 2;
3459 }
3460 break;
3461 } /* switch(cl) */
3462 } /* while (cl) */
3463
3464 if (ret != 0)
3465 sdbc_mark_cctl_cot(header, centry);
3466
3467 return (ret);
3468 }
3469
3470 /*
3471 * _sd_setup_mem_chaining(_sd_cctl_t *) - Allocate memory, setup
3472 * mem ptrs an host/pest chaining. Do the actual allocation as described in
3473 * sd_setup_category_on_type().
3474 *
3475 * RETURNS:
3476 * 0 on success
3477 * non-zero on error
3478 *
3479 * Note:
3480 * if called with ALLOC_NOWAIT, caller must check for non-zero return
3481 */
3482 static int
3483 _sd_setup_mem_chaining(_sd_cctl_t *header, int flag)
3484 {
3485 _sd_cctl_t *prev_ent, *next_ent, *centry;
3486 _sd_cctl_t *anchor = NULL;
3487 int cl, rc = 0;
3488
3489 ASSERT(header);
3490
3491 if (!header)
3492 return (0);
3493
3494 prev_ent = 0;
3495 centry = header;
3496 next_ent = centry->cc_chain;
3497 cl = 2;
3498 while (cl) {
3499 switch (cl) {
3500 case (1): /* chain to next/monitor for completion */
3501 centry->cc_aging_dm &= ~ELIGIBLE_ENTRY_DM;
3502 prev_ent = centry;
3503 centry = next_ent;
3504 next_ent = 0;
3505 cl = 0;
3506 if (centry) {
3507 next_ent = centry->cc_chain;
3508 cl = 2;
3509 }
3510 break;
3511
3512 case (2): /* vector to appropriate routine */
3513 if (centry->cc_aging_dm & HOST_ENTRY_DM)
3514 cl = 10;
3515 else if (centry->cc_aging_dm &
3516 PARASITIC_ENTRY_DM)
3517 cl = 15;
3518 else
3519 cl = 5;
3520 break;
3521
3522 case (5): /* OTHER processing - alloc mem */
3523 if (rc = sdbc_centry_memalloc_dm(centry,
3524 centry->cc_alloc_size_dm, flag))
3525 /* The allocation failed */
3526 cl = 0;
3527 else
3528 cl = 1;
3529 break;
3530
3531 /*
3532 * HOST entry processing - save the anchor pt,
3533 * alloc the memory,
3534 */
3535 case (10): /* setup head and nxt ptrs */
3536 anchor = centry;
3537 if (rc = sdbc_centry_memalloc_dm(centry,
3538 centry->cc_alloc_size_dm, flag))
3539 /* The allocation failed */
3540 cl = 0;
3541 else
3542 cl = 1;
3543 break;
3544
3545 /*
3546 * PARASITIC entry processing - setup w/no
3547 * memory, setup head/next ptrs,
3548 */
3549 case (15):
3550 /*
3551 * fudge the data mem ptr to an offset from
3552 * the anchor alloc
3553 */
3554 if (!(centry->cc_aging_dm &
3555 (HASH_ENTRY_DM| HOLD_ENTRY_DM))) {
3556 centry->cc_head_dm = anchor;
3557
3558 /* chain prev to this */
3559 prev_ent->cc_next_dm = centry;
3560
3561 /*
3562 * generate the actual data ptr into
3563 * host entry memory
3564 */
3565 centry->cc_data = anchor->cc_data +
3566 centry->cc_alloc_size_dm;
3567 centry->cc_alloc_size_dm = 0;
3568 }
3569 cl = 1;
3570 break;
3571 } /* switch(cl) */
3572 } /* while (cl) */
3573
3574 return (rc);
3575 }
3576
3577 /*
3578 * _sd_check_buffer_alloc - Check if buffer allocation is invalid.
3579 *
3580 * RETURNS:
3581 * 0 if its ok to continue with allocation.
3582 * Else errno to be returned to the user.
3583 *
3584 * Note:
3585 * This routine could block if the device is not local and
3586 * recovery is in progress.
3587 */
3588
3589 /* ARGSUSED */
3590 static int
3591 _sd_check_buffer_alloc(int cd, nsc_off_t fba_pos, nsc_size_t fba_len,
3592 _sd_buf_handle_t **hp)
3593 {
3594 /*
3595 * This check exists to ensure that someone will not pass in an
3596 * arbitrary pointer and try to pass it off as a handle.
3597 */
3598 if ((*hp)->bh_flag & (~_SD_VALID_FLAGS)) {
3599 cmn_err(CE_WARN, "!sdbc(_sd_check_buffer_alloc) "
3600 "cd %d invalid handle %p flags %x",
3601 cd, (void *)*hp, (*hp)->bh_flag);
3602 return (EINVAL);
3603 }
3604
3605 if ((_sd_cache_initialized == 0) || (FILE_OPENED(cd) == 0)) {
3606 cmn_err(CE_WARN, "!sdbc(_sd_check_buffer_alloc) "
3607 "cd %d not open. Cache init %d",
3608 cd, _sd_cache_initialized);
3609 return (EINVAL);
3610 }
3611 ASSERT(cd >= 0);
3612 if (!(_sd_cache_files[cd].cd_rawfd) ||
3613 !nsc_held(_sd_cache_files[cd].cd_rawfd)) {
3614 cmn_err(CE_WARN,
3615 "!sdbc(_sd_check_buffer_alloc) cd %d is not attached", cd);
3616 return (EINVAL);
3617 }
3618
3619 ASSERT_IO_SIZE(fba_pos, fba_len, cd);
3620 ASSERT_LEN(fba_len);
3621
3622 return (0);
3623 }
3624
3625 /*
3626 * sdbc_check_handle -- check that handle is valid
3627 * return 1 if ok, 0 otherwise (if debug then panic).
3628 */
3629 static int
3630 sdbc_check_handle(_sd_buf_handle_t *handle)
3631 {
3632 int ret = 1;
3633
3634 if (!_SD_HANDLE_ACTIVE(handle)) {
3635
3636 cmn_err(cmn_level, "!sdbc(_sd_free_buf): invalid handle %p"
3637 "cd %d fpos %" NSC_SZFMT " flen %" NSC_SZFMT " flag %x",
3638 (void *)handle, HANDLE_CD(handle), handle->bh_fba_pos,
3639 handle->bh_fba_len, handle->bh_flag);
3640
3641 ret = 0;
3642 }
3643
3644 return (ret);
3645 }
3646
3647 /*
3648 * _sd_free_buf - Free the buffers allocated in _sd_alloc_buf.
3649 *
3650 * ARGUMENTS:
3651 * handle - The handle allocated in _sd_alloc_buf.
3652 *
3653 * RETURNS:
3654 * 0 on success.
3655 * Else errno.
3656 *
3657 * NOTE:
3658 * If handle was allocated through _sd_alloc_buf, the handle allocated
3659 * flag (NSC_HALLOCATED) will be reset by _sd_alloc_buf. This indicates
3660 * that _sd_free_buf should free up the handle as well.
3661 * All other handles directly allocated from _sd_alloc_handle will have
3662 * that flag set. Any handle with valid blocks will have the handle
3663 * active flag. It is an error if the active flag is not set.
3664 * (if free_buf were called without going through alloc_buf)
3665 */
3666
3667 int
3668 _sd_free_buf(_sd_buf_handle_t *handle)
3669 {
3670 _sd_cctl_t *centry, *cc_chain;
3671 int cd = HANDLE_CD(handle);
3672 int flen = handle->bh_fba_len;
3673 int fpos = handle->bh_fba_pos;
3674
3675 SDTRACE(ST_ENTER|SDF_FREEBUF, HANDLE_CD(handle),
3676 handle->bh_fba_len, handle->bh_fba_pos, 0, 0);
3677
3678 if (sdbc_check_handle(handle) == 0)
3679 return (EINVAL);
3680
3681 if (handle->bh_flag & NSC_MIXED) {
3682 /*
3683 * Data in this handle will be a mix of data from the
3684 * source device and data from another device, so
3685 * invalidate all the blocks.
3686 */
3687 handle->bh_flag &= ~NSC_QUEUE;
3688 centry = handle->bh_centry;
3689 while (centry) {
3690 centry->cc_valid = 0;
3691 centry = centry->cc_chain;
3692 }
3693 }
3694
3695 if ((handle->bh_flag & NSC_QUEUE)) {
3696 handle->bh_flag &= ~NSC_QUEUE;
3697 _sd_queue_write(handle, handle->bh_fba_pos, handle->bh_fba_len);
3698 }
3699
3700 handle->bh_flag &= ~NSC_HACTIVE;
3701
3702 centry = handle->bh_centry;
3703 while (centry) {
3704 cc_chain = centry->cc_chain;
3705 _sd_centry_release(centry);
3706 centry = cc_chain;
3707 }
3708
3709 /*
3710 * help prevent dup call to _sd_centry_release if this handle
3711 * is erroneously _sd_free_buf'd twice. (should not happen).
3712 */
3713 handle->bh_centry = NULL;
3714
3715 if ((handle->bh_flag & NSC_HALLOCATED) == 0) {
3716 handle->bh_flag |= NSC_HALLOCATED;
3717 (void) _sd_free_handle(handle);
3718 } else {
3719 handle->bh_flag = NSC_HALLOCATED;
3720 }
3721
3722 SDTRACE(ST_EXIT|SDF_FREEBUF, cd, flen, fpos, 0, 0);
3723
3724 return (0);
3725 }
3726
3727
3728 static int _sd_lruq_srch = 0x2000;
3729
3730 /*
3731 * sdbc_get_dmchain -- get a candidate centry chain pointing to
3732 * contiguous memory
3733 * ARGUMENTS:
3734 * cblocks - number of cache blocks requested
3735 * stall - pointer to stall count (no blocks avail)
3736 * flag - ALLOC_NOWAIT flag
3737 *
3738 * RETURNS:
3739 * a cache entry or possible NULL if ALLOC_NOWAIT set
3740 * USAGE:
3741 * attempt to satisfy entire request from queue
3742 * that has no memory allocated.
3743 * if this fails then attempt a partial allocation
3744 * with a preallocated block of requested size up to
3745 * max_dyn_list.
3746 * then look for largest chain less than max_dyn_list.
3747 */
3748 static _sd_cctl_t *
3749 sdbc_get_dmchain(int cblocks, int *stall, int flag)
3750 {
3751 _sd_cctl_t *cc_dmchain = NULL;
3752 _sd_queue_t *q;
3753 _sd_cctl_t *qhead;
3754 int num_tries;
3755 int cblocks_orig = cblocks;
3756 int nowait = flag & ALLOC_NOWAIT;
3757 int i;
3758
3759 num_tries = _sd_lruq_srch;
3760
3761 ASSERT(cblocks != 0);
3762
3763 while (!cc_dmchain) {
3764 /* get it from the os if possible */
3765 q = &sdbc_dm_queues[0];
3766 qhead = &(q->sq_qhead);
3767
3768 if (q->sq_inq >= cblocks) {
3769 mutex_enter(&q->sq_qlock);
3770 if (q->sq_inq >= cblocks) {
3771 _sd_cctl_t *cc_ent;
3772
3773 cc_dmchain = qhead->cc_next;
3774
3775 /*
3776 * set the inuse and pageio bits
3777 * Note: this code expects the cc_ent to
3778 * be available. no other thread may set the
3779 * inuse or pageio bit for an entry on the
3780 * 0 queue.
3781 */
3782 cc_ent = qhead;
3783 for (i = 0; i < cblocks; ++i) {
3784 cc_ent = cc_ent->cc_next;
3785
3786 if (SET_CENTRY_INUSE(cc_ent)) {
3787 cmn_err(CE_PANIC,
3788 "centry inuse on 0 q! %p",
3789 (void *)cc_ent);
3790 }
3791
3792 if (SET_CENTRY_PAGEIO(cc_ent)) {
3793 cmn_err(CE_PANIC,
3794 "centry pageio on 0 q! %p",
3795 (void *)cc_ent);
3796 }
3797 }
3798 /* got a dmchain */
3799
3800 /* remove this chain from the 0 queue */
3801 cc_dmchain->cc_prev->cc_next = cc_ent->cc_next;
3802 cc_ent->cc_next->cc_prev = cc_dmchain->cc_prev;
3803 cc_dmchain->cc_prev = NULL;
3804 cc_ent->cc_next = NULL;
3805
3806 q->sq_inq -= cblocks;
3807
3808 ASSERT(GOOD_LRUSIZE(q));
3809
3810 }
3811 mutex_exit(&q->sq_qlock);
3812 if (cc_dmchain)
3813 continue;
3814 }
3815
3816 /* look for a pre-allocated block of the requested size */
3817
3818
3819 if (cblocks > (max_dm_queues - 1))
3820 cblocks = max_dm_queues - 1;
3821
3822 q = &sdbc_dm_queues[cblocks];
3823 qhead = &(q->sq_qhead);
3824
3825 if (q->sq_inq != 0) {
3826 _sd_cctl_t *tmp_dmchain;
3827
3828 mutex_enter(&q->sq_qlock);
3829
3830 for (tmp_dmchain = qhead->cc_next; tmp_dmchain != qhead;
3831 tmp_dmchain = tmp_dmchain->cc_next) {
3832
3833 /*
3834 * get a dmchain
3835 * set the inuse and pageio bits
3836 */
3837 if (sdbc_dmchain_avail(tmp_dmchain)) {
3838 /* put on MRU end of queue */
3839 sdbc_requeue_dmchain(q, tmp_dmchain,
3840 1, 0);
3841 cc_dmchain = tmp_dmchain;
3842 break;
3843 }
3844 sdbc_dmchain_not_avail++;
3845 }
3846
3847 mutex_exit(&q->sq_qlock);
3848 if (cc_dmchain)
3849 continue;
3850 }
3851
3852 /*
3853 * spin block
3854 * nudge the deallocator, accelerate ageing
3855 */
3856
3857 mutex_enter(&dynmem_processing_dm.thread_dm_lock);
3858 cv_broadcast(&dynmem_processing_dm.thread_dm_cv);
3859 mutex_exit(&dynmem_processing_dm.thread_dm_lock);
3860
3861 if (nowait)
3862 break;
3863
3864 if (!(--num_tries)) {
3865 delay(drv_usectohz(20000));
3866 (void) (*stall)++;
3867 num_tries = _sd_lruq_srch;
3868 cblocks = cblocks_orig;
3869 } else { /* see if smaller request size is available */
3870 if (!(--cblocks))
3871 cblocks = cblocks_orig;
3872 }
3873
3874 } /* while (!cc_dmchain) */
3875
3876 return (cc_dmchain);
3877 }
3878
3879 static int
3880 sdbc_dmchain_avail(_sd_cctl_t *cc_ent)
3881 {
3882 int chain_avail = 1;
3883 _sd_cctl_t *anchor = cc_ent;
3884
3885 while (cc_ent) {
3886
3887 ASSERT(_sd_cctl_valid(cc_ent));
3888
3889 if (cc_ent->cc_aging_dm & BAD_CHAIN_DM) {
3890 chain_avail = 0;
3891 break;
3892 }
3893
3894 if (CENTRY_DIRTY(cc_ent)) {
3895 chain_avail = 0;
3896 break;
3897 }
3898 if (SET_CENTRY_INUSE(cc_ent)) {
3899 chain_avail = 0;
3900 break;
3901 }
3902
3903 if ((SET_CENTRY_PAGEIO(cc_ent))) {
3904
3905 CLEAR_CENTRY_INUSE(cc_ent);
3906 chain_avail = 0;
3907 break;
3908 }
3909
3910 if (CENTRY_DIRTY(cc_ent)) {
3911
3912 CLEAR_CENTRY_PAGEIO(cc_ent);
3913 CLEAR_CENTRY_INUSE(cc_ent);
3914 chain_avail = 0;
3915 break;
3916 }
3917
3918 cc_ent->cc_flag = 0;
3919 cc_ent->cc_toflush = 0;
3920
3921 cc_ent = cc_ent->cc_next_dm;
3922 }
3923
3924 if (!chain_avail)
3925 sdbc_clear_dmchain(anchor, cc_ent);
3926 else {
3927 cc_ent = anchor;
3928
3929 /*
3930 * prevent possible deadlocks in _sd_cc_wait():
3931 * remove from hash and wakeup any waiters now that we
3932 * have acquired the chain.
3933 */
3934 while (cc_ent) {
3935 (void) _sd_hash_delete((struct _sd_hash_hd *)cc_ent,
3936 _sd_htable);
3937
3938 mutex_enter(&cc_ent->cc_lock);
3939 if (cc_ent->cc_await_use) {
3940 cv_broadcast(&cc_ent->cc_blkcv);
3941 }
3942 mutex_exit(&cc_ent->cc_lock);
3943
3944 cc_ent->cc_creat = nsc_lbolt();
3945 cc_ent->cc_hits = 0;
3946
3947 cc_ent = cc_ent->cc_next_dm;
3948 }
3949 }
3950
3951 return (chain_avail);
3952 }
3953
3954 static void
3955 sdbc_clear_dmchain(_sd_cctl_t *cc_ent_start, _sd_cctl_t *cc_ent_end)
3956 {
3957 _sd_cctl_t *cc_ent = cc_ent_start;
3958 _sd_cctl_t *prev_ent;
3959
3960 ASSERT(_sd_cctl_valid(cc_ent));
3961
3962 while (cc_ent != cc_ent_end) {
3963
3964 ASSERT(_sd_cctl_valid(cc_ent));
3965
3966 prev_ent = cc_ent;
3967 cc_ent = cc_ent->cc_next_dm;
3968
3969 CLEAR_CENTRY_PAGEIO(prev_ent);
3970 CLEAR_CENTRY_INUSE(prev_ent);
3971 }
3972
3973 }
3974
3975 /*
3976 * put a dmchain on the LRU end of a queue
3977 */
3978 void
3979 sdbc_ins_dmqueue_front(_sd_queue_t *q, _sd_cctl_t *cc_ent)
3980 {
3981 _sd_cctl_t *qhead = &(q->sq_qhead);
3982
3983 ASSERT(_sd_cctl_valid(cc_ent));
3984
3985 mutex_enter(&q->sq_qlock);
3986 cc_ent->cc_next = qhead->cc_next;
3987 cc_ent->cc_prev = qhead;
3988 qhead->cc_next->cc_prev = cc_ent;
3989 qhead->cc_next = cc_ent;
3990 q->sq_inq++;
3991 cc_ent->cc_cblocks = q->sq_dmchain_cblocks;
3992
3993 ASSERT(GOOD_LRUSIZE(q));
3994
3995 mutex_exit(&q->sq_qlock);
3996
3997 }
3998
3999 /*
4000 * put a dmchain on the MRU end of a queue
4001 */
4002 static void
4003 sdbc_ins_dmqueue_back(_sd_queue_t *q, _sd_cctl_t *cc_ent)
4004 {
4005 _sd_cctl_t *qhead = &(q->sq_qhead);
4006
4007 ASSERT(_sd_cctl_valid(cc_ent));
4008
4009 mutex_enter(&q->sq_qlock);
4010 cc_ent->cc_next = qhead;
4011 cc_ent->cc_prev = qhead->cc_prev;
4012 qhead->cc_prev->cc_next = cc_ent;
4013 qhead->cc_prev = cc_ent;
4014 cc_ent->cc_seq = q->sq_seq++;
4015 q->sq_inq++;
4016 cc_ent->cc_cblocks = q->sq_dmchain_cblocks;
4017
4018 ASSERT(GOOD_LRUSIZE(q));
4019
4020 mutex_exit(&q->sq_qlock);
4021
4022 }
4023
4024 /*
4025 * remove dmchain from a queue
4026 */
4027 void
4028 sdbc_remq_dmchain(_sd_queue_t *q, _sd_cctl_t *cc_ent)
4029 {
4030
4031 ASSERT(_sd_cctl_valid(cc_ent));
4032
4033 mutex_enter(&q->sq_qlock);
4034 cc_ent->cc_prev->cc_next = cc_ent->cc_next;
4035 cc_ent->cc_next->cc_prev = cc_ent->cc_prev;
4036 cc_ent->cc_next = cc_ent->cc_prev = NULL; /* defensive programming */
4037 cc_ent->cc_cblocks = -1; /* indicate not on any queue */
4038
4039 q->sq_inq--;
4040
4041 ASSERT(GOOD_LRUSIZE(q));
4042
4043 mutex_exit(&q->sq_qlock);
4044
4045 }
4046
4047 /*
4048 * requeue a dmchain to the MRU end of its queue.
4049 * if getlock is 0 on entry the queue lock (sq_qlock) must be held
4050 */
4051 void
4052 sdbc_requeue_dmchain(_sd_queue_t *q, _sd_cctl_t *cc_ent, int mru,
4053 int getlock)
4054 {
4055 _sd_cctl_t *qhead = &(q->sq_qhead);
4056
4057
4058 ASSERT(_sd_cctl_valid(cc_ent));
4059
4060 if (getlock)
4061 mutex_enter(&q->sq_qlock);
4062
4063 /* inline of sdbc_remq_dmchain() */
4064 cc_ent->cc_prev->cc_next = cc_ent->cc_next;
4065 cc_ent->cc_next->cc_prev = cc_ent->cc_prev;
4066
4067 if (mru) { /* put on MRU end of queue */
4068 /* inline of sdbc_ins_dmqueue_back */
4069 cc_ent->cc_next = qhead;
4070 cc_ent->cc_prev = qhead->cc_prev;
4071 qhead->cc_prev->cc_next = cc_ent;
4072 qhead->cc_prev = cc_ent;
4073 cc_ent->cc_seq = q->sq_seq++;
4074 (q->sq_req_stat)++;
4075 } else { /* put on LRU end of queue i.e. requeue to head */
4076 /* inline of sdbc_ins_dmqueue_front */
4077 cc_ent->cc_next = qhead->cc_next;
4078 cc_ent->cc_prev = qhead;
4079 qhead->cc_next->cc_prev = cc_ent;
4080 qhead->cc_next = cc_ent;
4081 cc_ent->cc_seq = q->sq_seq++;
4082
4083 /*
4084 * clear the CC_QHEAD bit on all members of the chain
4085 */
4086 {
4087 _sd_cctl_t *tcent;
4088
4089 for (tcent = cc_ent; tcent; tcent = tcent->cc_next_dm)
4090 tcent->cc_flag &= ~CC_QHEAD;
4091 }
4092 }
4093
4094 if (getlock)
4095 mutex_exit(&q->sq_qlock);
4096
4097 }
4098
4099 /*
4100 * sdbc_dmchain_dirty(cc_ent)
4101 * return first dirty cc_ent in dmchain, NULL if chain is not dirty
4102 */
4103 static _sd_cctl_t *
4104 sdbc_dmchain_dirty(_sd_cctl_t *cc_ent)
4105 {
4106 for (/* CSTYLED */; cc_ent; cc_ent = cc_ent->cc_next_dm)
4107 if (CENTRY_DIRTY(cc_ent))
4108 break;
4109
4110 return (cc_ent);
4111 }
4112
4113 /*
4114 * sdbc_requeue_head_dm_try()
4115 * attempt to requeue a dmchain to the head of the queue
4116 */
4117 void
4118 sdbc_requeue_head_dm_try(_sd_cctl_t *cc_ent)
4119 {
4120 int qidx;
4121 _sd_queue_t *q;
4122
4123 if (!sdbc_dmchain_dirty(cc_ent)) {
4124 qidx = cc_ent->cc_cblocks;
4125 q = &sdbc_dm_queues[qidx];
4126 sdbc_requeue_dmchain(q, cc_ent, 0, 1); /* requeue head */
4127 }
4128 }
4129
4130 /*
4131 * sdbc_centry_alloc_blks -- allocate cache entries with memory
4132 *
4133 * ARGUMENTS:
4134 * cd - Cache descriptor (from a previous open)
4135 * cblk - cache block number.
4136 * reqblks - number of cache blocks to be allocated
4137 * flag - can be ALLOC_NOWAIT
4138 * RETURNS:
4139 * A cache block chain or NULL if ALLOC_NOWAIT and request fails
4140 *
4141 * Note: caller must check for null return if called with
4142 * ALLOC_NOWAIT set.
4143 */
4144 _sd_cctl_t *
4145 sdbc_centry_alloc_blks(int cd, nsc_off_t cblk, nsc_size_t reqblks, int flag)
4146 {
4147 sdbc_allocbuf_t alloc_tok = {0}; /* must be 0 */
4148 int stall = 0;
4149 _sd_cctl_t *centry = NULL;
4150 _sd_cctl_t *lentry = NULL;
4151 _sd_cctl_t *anchor = NULL;
4152 _sd_cctl_t *next_centry;
4153
4154 ASSERT(reqblks);
4155
4156 while (reqblks) {
4157 centry = sdbc_centry_alloc(cd, cblk, reqblks, &stall,
4158 &alloc_tok, flag);
4159
4160 if (!centry)
4161 break;
4162
4163 centry->cc_chain = NULL;
4164
4165 if (lentry == NULL)
4166 anchor = centry;
4167 else
4168 lentry->cc_chain = centry;
4169
4170 lentry = centry;
4171
4172 centry->cc_aging_dm &= ~(ENTRY_FIELD_DM);
4173
4174 if (centry->cc_aging_dm & FOUND_IN_HASH_DM)
4175 centry->cc_aging_dm |= HASH_ENTRY_DM;
4176 else
4177 if (centry->cc_aging_dm & FOUND_HOLD_OVER_DM)
4178 centry->cc_aging_dm |= HOLD_ENTRY_DM;
4179 else
4180 centry->cc_aging_dm |= ELIGIBLE_ENTRY_DM;
4181
4182 centry->cc_aging_dm &= ~(FOUND_IN_HASH_DM|FOUND_HOLD_OVER_DM);
4183 --reqblks;
4184 }
4185
4186 sdbc_centry_alloc_end(&alloc_tok);
4187
4188 if (reqblks || (_sd_setup_category_on_type(anchor))) {
4189 centry = anchor;
4190 while (centry) {
4191 next_centry = centry->cc_chain;
4192 _sd_centry_release(centry);
4193 centry = next_centry;
4194 }
4195 anchor = NULL;
4196
4197 } else
4198 /* This is where the memory is actually allocated */
4199 if (_sd_setup_mem_chaining(anchor, flag))
4200 anchor = NULL;
4201
4202 return (anchor);
4203 }
4204
4205
4206 /*
4207 * sdbc_centry_alloc - sdbc internal function to allocate a new cache block.
4208 *
4209 * ARGUMENTS:
4210 * cd - Cache descriptor (from a previous open)
4211 * cblk - cache block number.
4212 * stall - pointer to stall count (no blocks avail)
4213 * req_blocks - number of cache blocks remaining in caller's i/o request
4214 * alloc_tok - pointer to token initialized to 0 on first call to function
4215 * flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT flag
4216 * RETURNS:
4217 * A cache block, or possibly NULL if ALLOC_NOWAIT set .
4218 *
4219 * USAGE:
4220 * switch to the appropriate allocation function.
4221 * this function is used when callers need more than one cache block.
4222 * it is called repeatedly until the entire request is satisfied,
4223 * at which time the caller will then do the memory allocation.
4224 * if only one cache block is needed callers may use
4225 * sdbc_centry_alloc_blks() which also allocates memory.
4226 *
4227 * Note: caller must check for null return if called with
4228 * ALLOC_NOWAIT set.
4229 */
4230
4231 _sd_cctl_t *
4232 sdbc_centry_alloc(int cd, nsc_off_t cblk, nsc_size_t req_blocks, int *stall,
4233 sdbc_allocbuf_t *alloc_tok, int flag)
4234 {
4235 _sd_cctl_t *centry;
4236
4237 if (sdbc_use_dmchain)
4238 centry = sdbc_alloc_dmc(cd, cblk, req_blocks, stall, alloc_tok,
4239 flag);
4240 else
4241 centry = sdbc_alloc_lru(cd, cblk, stall, flag);
4242
4243 return (centry);
4244 }
4245
4246 /*
4247 * sdbc_alloc_dmc -- allocate a centry from a dmchain
4248 *
4249 * ARGUMENTS:
4250 * cd - Cache descriptor (from a previous open)
4251 * cblk - cache block number.
4252 * stall - pointer to stall count (no blocks avail)
4253 * req_blocks - number of cache blocks in clients i/o request
4254 * alloc_tok - pointer to token initialized to 0 on first call to function
4255 * flag - lock status of sdbc_queue_lock, or ALLOC_NOWAIT flag
4256 * RETURNS:
4257 * A cache block or possibly NULL if ALLOC_NOWAIT set
4258 *
4259 * USAGE:
4260 * if dmchain is empty, allocate one.
4261 */
4262 static _sd_cctl_t *
4263 sdbc_alloc_dmc(int cd, nsc_off_t cblk, nsc_size_t req_blocks, int *stall,
4264 sdbc_allocbuf_t *alloc_tok, int flag)
4265 {
4266 sdbc_allocbuf_impl_t *dmc = (sdbc_allocbuf_impl_t *)alloc_tok;
4267 _sd_cctl_t *centry = NULL;
4268
4269 if (!dmc->sab_dmchain) {
4270 /*
4271 * Note - sdbc_get_dmchain() returns
4272 * with cc_inuse and cc_pageio set
4273 * for all members of dmchain.
4274 */
4275 if (dmc->sab_dmchain =
4276 sdbc_get_dmchain(req_blocks, stall, flag)) {
4277
4278 /* remember q it came from */
4279 if (dmc->sab_dmchain->cc_alloc_size_dm)
4280 dmc->sab_q = dmc->sab_dmchain->cc_cblocks;
4281 }
4282 }
4283
4284 /*
4285 * Note: dmchain pointer is advanced in sdbc_alloc_from_dmchain()
4286 */
4287 if (dmc->sab_dmchain) /* could be NULL if ALLOC_NOWAIT set */
4288 centry = sdbc_alloc_from_dmchain(cd, cblk, alloc_tok, flag);
4289
4290 return (centry);
4291 }
4292
4293 /*
4294 * sdbc_alloc_from_dmchain -- allocate centry from a dmchain of centrys
4295 *
4296 * ARGUMENTS:
4297 * cd - Cache descriptor (from a previous open)
4298 * cblk - cache block number.
4299 * alloc_tok - pointer to token
4300 * flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT
4301 *
4302 * RETURNS:
4303 * A cache block or possibly NULL if ALLOC_NOWAIT set.
4304 *
4305 * USAGE:
4306 * This routine allocates a new cache block from the supplied dmchain.
4307 * Assumes that dmchain is non-NULL and that all cache entries in
4308 * the dmchain have been removed from hash and have their cc_inuse and
4309 * cc_pageio bits set.
4310 */
4311 static _sd_cctl_t *
4312 sdbc_alloc_from_dmchain(int cd, nsc_off_t cblk, sdbc_allocbuf_t *alloc_tok,
4313 int flag)
4314 {
4315 _sd_cctl_t *cc_ent, *old_ent;
4316 int categorize_centry;
4317 int locked = flag & ALLOC_LOCKED;
4318 int nowait = flag & ALLOC_NOWAIT;
4319 sdbc_allocbuf_impl_t *dmc = (sdbc_allocbuf_impl_t *)alloc_tok;
4320
4321 SDTRACE(ST_ENTER|SDF_ENT_ALLOC, cd, 0, BLK_TO_FBA_NUM(cblk), 0, 0);
4322
4323 ASSERT(dmc->sab_dmchain);
4324
4325 cc_ent = dmc->sab_dmchain;
4326
4327 ASSERT(_sd_cctl_valid(cc_ent));
4328
4329 cc_ent->cc_valid = 0;
4330 categorize_centry = 0;
4331 if (cc_ent->cc_data)
4332 categorize_centry = FOUND_HOLD_OVER_DM;
4333
4334 alloc_try:
4335 if (cd == _CD_NOHASH)
4336 CENTRY_BLK(cc_ent) = cblk;
4337 else if ((old_ent = (_sd_cctl_t *)
4338 _sd_hash_insert(cd, cblk, (struct _sd_hash_hd *)cc_ent,
4339 _sd_htable)) != cc_ent) {
4340
4341 if (SET_CENTRY_INUSE(old_ent)) {
4342 sdbc_centry_inuse++;
4343
4344 if (nowait) {
4345 cc_ent = NULL;
4346 goto out;
4347 }
4348
4349 if (locked)
4350 rw_exit(&sdbc_queue_lock);
4351 _sd_cc_wait(cd, cblk, old_ent, CC_INUSE);
4352 if (locked)
4353 rw_enter(&sdbc_queue_lock, RW_WRITER);
4354 goto alloc_try;
4355 }
4356
4357 /*
4358 * bug 4529671
4359 * now that we own the centry make sure that
4360 * it is still good. it could have been processed
4361 * by _sd_dealloc_dm() in the window between
4362 * _sd_hash_insert() and SET_CENTRY_INUSE().
4363 */
4364 if ((_sd_cctl_t *)_sd_hash_search(cd, cblk, _sd_htable)
4365 != old_ent) {
4366 sdbc_centry_deallocd++;
4367 #ifdef DEBUG
4368 cmn_err(CE_WARN, "!cc_ent %p cd %d cblk %" NSC_SZFMT
4369 " lost to dealloc?! cc_data %p", (void *)old_ent,
4370 cd, cblk, (void *)old_ent->cc_data);
4371 #endif
4372
4373 CLEAR_CENTRY_INUSE(old_ent);
4374
4375 if (nowait) {
4376 cc_ent = NULL;
4377 goto out;
4378 }
4379
4380 goto alloc_try;
4381 }
4382
4383 if (CC_CD_BLK_MATCH(cd, cblk, old_ent)) {
4384 sdbc_centry_hit++;
4385 old_ent->cc_toflush = 0;
4386 /* _sd_centry_release(cc_ent); */
4387 cc_ent = old_ent;
4388 categorize_centry = FOUND_IN_HASH_DM;
4389 } else {
4390 sdbc_centry_lost++;
4391
4392 CLEAR_CENTRY_INUSE(old_ent);
4393
4394 if (nowait) {
4395 cc_ent = NULL;
4396 goto out;
4397 }
4398
4399 goto alloc_try;
4400 }
4401 }
4402
4403 /*
4404 * advance the dmchain pointer, but only if we got the
4405 * cc_ent from the dmchain
4406 */
4407 if (categorize_centry != FOUND_IN_HASH_DM) {
4408 if (cc_ent->cc_data)
4409 dmc->sab_dmchain = dmc->sab_dmchain->cc_next_dm;
4410 else
4411 dmc->sab_dmchain = dmc->sab_dmchain->cc_next;
4412 }
4413
4414
4415 SDTRACE(ST_EXIT|SDF_ENT_ALLOC, cd, 0, BLK_TO_FBA_NUM(cblk), 0, 0);
4416
4417 mutex_enter(&cc_ent->cc_lock);
4418 if (cc_ent->cc_await_use) {
4419 cv_broadcast(&cc_ent->cc_blkcv);
4420 }
4421 mutex_exit(&cc_ent->cc_lock);
4422
4423 sdbc_centry_init_dm(cc_ent);
4424
4425 cc_ent->cc_aging_dm |= categorize_centry;
4426
4427 out:
4428
4429 SDTRACE(ST_INFO|SDF_ENT_ALLOC, cd, 0, BLK_TO_FBA_NUM(cblk), 0, 0);
4430
4431 return (cc_ent);
4432 }
4433
4434 /*
4435 * sdbc_centry_alloc_end -- tidy up after all cache blocks have been
4436 * allocated for a request
4437 * ARGUMENTS:
4438 * alloc_tok - pointer to allocation token
4439 * RETURNS
4440 * nothing
4441 * USAGE:
4442 * at this time only useful when sdbc_use_dmchain is true.
4443 * if there are cache blocks remaining on the chain then the inuse and
4444 * pageio bits must be cleared (they were set in sdbc_get_dmchain().
4445 *
4446 */
4447 static void
4448 sdbc_centry_alloc_end(sdbc_allocbuf_t *alloc_tok)
4449 {
4450 _sd_cctl_t *next_centry;
4451 _sd_cctl_t *prev_centry;
4452 _sd_queue_t *q;
4453 sdbc_allocbuf_impl_t *dmc = (sdbc_allocbuf_impl_t *)alloc_tok;
4454 #ifdef DEBUG
4455 int chainpull = 0;
4456 #endif
4457
4458 if (!sdbc_use_dmchain)
4459 return;
4460
4461 next_centry = dmc->sab_dmchain;
4462
4463 while (next_centry != NULL) {
4464 CLEAR_CENTRY_PAGEIO(next_centry);
4465
4466 prev_centry = next_centry;
4467
4468 if (next_centry->cc_data) {
4469 #ifdef DEBUG
4470 ++chainpull;
4471 #endif
4472 next_centry = next_centry->cc_next_dm;
4473
4474 /* clear bit after final reference */
4475
4476 CLEAR_CENTRY_INUSE(prev_centry);
4477 } else {
4478 next_centry = next_centry->cc_next;
4479
4480 /*
4481 * a floater from the 0 queue, insert on q.
4482 *
4483 * since this centry is not on any queue
4484 * the inuse bit can be cleared before
4485 * inserting on the q. this is also required
4486 * since sdbc_get_dmchain() does not expect
4487 * inuse bits to be set on 0 queue entry's.
4488 */
4489
4490 CLEAR_CENTRY_INUSE(prev_centry);
4491 q = &sdbc_dm_queues[0];
4492 sdbc_ins_dmqueue_front(q, prev_centry);
4493 }
4494 }
4495
4496 #ifdef DEBUG
4497 /* compute wastage stats */
4498 ASSERT((chainpull >= 0) && (chainpull < max_dm_queues));
4499 if (chainpull)
4500 (*(dmchainpull_table + (dmc->sab_q *
4501 max_dm_queues + chainpull)))++;
4502 #endif
4503
4504 }
4505
4506
4507 /*
4508 * sdbc_alloc_lru - allocate a new cache block from the lru queue
4509 *
4510 * ARGUMENTS:
4511 * cd - Cache descriptor (from a previous open)
4512 * cblk - cache block number.
4513 * stall - pointer to stall count (no blocks avail)
4514 * flag - lock status of sdbc_queue_lock or ALLOC_NOWAIT
4515 *
4516 * RETURNS:
4517 * A cache block or NULL if ALLOC_NOWAIT specified
4518 *
4519 * USAGE:
4520 * This routine allocates a new cache block from the lru.
4521 * If an allocation cannot be done, we block, unless ALLOC_NOWAIT is set.
4522 */
4523
4524 static _sd_cctl_t *
4525 sdbc_alloc_lru(int cd, nsc_off_t cblk, int *stall, int flag)
4526 {
4527 _sd_cctl_t *cc_ent, *old_ent, *ccnext;
4528 _sd_queue_t *q = _SD_LRU_Q;
4529 _sd_cctl_t *qhead = &(q->sq_qhead);
4530 int tries = 0, num_tries;
4531 int categorize_centry;
4532 int locked = flag & ALLOC_LOCKED;
4533 int nowait = flag & ALLOC_NOWAIT;
4534
4535 if (nowait) {
4536 num_tries = q->sq_inq / 100; /* only search 1% of q */
4537
4538 if (num_tries <= 0) /* ensure num_tries is non-zero */
4539 num_tries = q->sq_inq;
4540 } else
4541 num_tries = _sd_lruq_srch;
4542
4543 SDTRACE(ST_ENTER|SDF_ENT_ALLOC, cd, 0, BLK_TO_FBA_NUM(cblk), 0, 0);
4544 retry_alloc_centry:
4545
4546 for (cc_ent = (qhead->cc_next); cc_ent != qhead; cc_ent = ccnext) {
4547 if (--num_tries <= 0)
4548 if (nowait) {
4549 cc_ent = NULL;
4550 goto out;
4551 } else
4552 break;
4553
4554 ccnext = cc_ent->cc_next;
4555
4556 if (cc_ent->cc_aging_dm & BAD_CHAIN_DM)
4557 continue;
4558
4559 if (CENTRY_DIRTY(cc_ent))
4560 continue;
4561 if (SET_CENTRY_INUSE(cc_ent))
4562 continue;
4563
4564 if (CENTRY_DIRTY(cc_ent)) {
4565 sdbc_centry_lost++;
4566
4567 CLEAR_CENTRY_INUSE(cc_ent);
4568 continue;
4569 }
4570 cc_ent->cc_flag = 0; /* CC_INUSE */
4571 cc_ent->cc_toflush = 0;
4572
4573 /*
4574 * Inlined requeue of the LRU. (should match _sd_requeue)
4575 */
4576 /* was FAST */
4577 mutex_enter(&q->sq_qlock);
4578 #if defined(_SD_DEBUG)
4579 if (1) {
4580 _sd_cctl_t *cp, *cn, *qp;
4581 cp = cc_ent->cc_prev;
4582 cn = cc_ent->cc_next;
4583 qp = (q->sq_qhead).cc_prev;
4584 if (!_sd_cctl_valid(cc_ent) ||
4585 (cp != &(q->sq_qhead) && !_sd_cctl_valid(cp)) ||
4586 (cn != &(q->sq_qhead) && !_sd_cctl_valid(cn)) ||
4587 !_sd_cctl_valid(qp))
4588 cmn_err(CE_PANIC,
4589 "_sd_centry_alloc %x prev %x next %x qp %x",
4590 cc_ent, cp, cn, qp);
4591 }
4592 #endif
4593 cc_ent->cc_prev->cc_next = cc_ent->cc_next;
4594 cc_ent->cc_next->cc_prev = cc_ent->cc_prev;
4595 cc_ent->cc_next = qhead;
4596 cc_ent->cc_prev = qhead->cc_prev;
4597 qhead->cc_prev->cc_next = cc_ent;
4598 qhead->cc_prev = cc_ent;
4599 cc_ent->cc_seq = q->sq_seq++;
4600 /* was FAST */
4601 mutex_exit(&q->sq_qlock);
4602 /*
4603 * End inlined requeue.
4604 */
4605
4606 #if defined(_SD_STATS)
4607 if (_sd_hash_delete(cc_ent, _sd_htable) == 0)
4608 SDTRACE(SDF_REPLACE,
4609 CENTRY_CD(cc_ent), cc_ent->cc_hits,
4610 BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)),
4611 nsc_lbolt(), cc_ent->cc_creat);
4612 cc_ent->cc_creat = nsc_lbolt();
4613 cc_ent->cc_hits = 0;
4614 #else
4615 #if defined(_SD_DEBUG)
4616 if (_sd_hash_delete(cc_ent, _sd_htable) == 0) {
4617 SDTRACE(SDF_REPLACE|ST_DL,
4618 CENTRY_CD(cc_ent),
4619 cc_ent->cc_valid,
4620 BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)),
4621 cd, BLK_TO_FBA_NUM(cblk));
4622 if (cc_ent->cc_await_use ||
4623 ((cd == CENTRY_CD(cc_ent)) &&
4624 (cblk == CENTRY_BLK(cc_ent))))
4625 DATA_LOG(SDF_REPLACE|ST_DL, cc_ent, 0,
4626 BLK_FBAS);
4627 }
4628 #else
4629 (void) _sd_hash_delete((struct _sd_hash_hd *)cc_ent,
4630 _sd_htable);
4631 #endif
4632 #endif
4633 cc_ent->cc_creat = nsc_lbolt();
4634 cc_ent->cc_hits = 0;
4635
4636 cc_ent->cc_valid = 0;
4637 categorize_centry = 0;
4638 if (cc_ent->cc_data)
4639 categorize_centry = FOUND_HOLD_OVER_DM;
4640
4641 alloc_try:
4642 if (cd == _CD_NOHASH)
4643 CENTRY_BLK(cc_ent) = cblk;
4644 else if ((old_ent = (_sd_cctl_t *)
4645 _sd_hash_insert(cd, cblk, (struct _sd_hash_hd *)cc_ent,
4646 _sd_htable)) != cc_ent) {
4647
4648 if (SET_CENTRY_INUSE(old_ent)) {
4649 sdbc_centry_inuse++;
4650
4651 if (nowait) {
4652 _sd_centry_release(cc_ent);
4653 cc_ent = NULL;
4654 goto out;
4655 }
4656
4657 if (locked)
4658 rw_exit(&sdbc_queue_lock);
4659 _sd_cc_wait(cd, cblk, old_ent, CC_INUSE);
4660 if (locked)
4661 rw_enter(&sdbc_queue_lock, RW_WRITER);
4662 goto alloc_try;
4663 }
4664
4665 /*
4666 * bug 4529671
4667 * now that we own the centry make sure that
4668 * it is still good. it could have been processed
4669 * by _sd_dealloc_dm() in the window between
4670 * _sd_hash_insert() and SET_CENTRY_INUSE().
4671 */
4672 if ((_sd_cctl_t *)
4673 _sd_hash_search(cd, cblk, _sd_htable) != old_ent) {
4674 sdbc_centry_deallocd++;
4675 #ifdef DEBUG
4676 cmn_err(CE_WARN, "!cc_ent %p cd %d cblk %"
4677 NSC_SZFMT " lost to dealloc?! cc_data %p",
4678 (void *)old_ent, cd, cblk,
4679 (void *)old_ent->cc_data);
4680 #endif
4681
4682 CLEAR_CENTRY_INUSE(old_ent);
4683
4684 if (nowait) {
4685 _sd_centry_release(cc_ent);
4686 cc_ent = NULL;
4687 goto out;
4688 }
4689
4690 goto alloc_try;
4691 }
4692
4693 if (CC_CD_BLK_MATCH(cd, cblk, old_ent)) {
4694 sdbc_centry_hit++;
4695 old_ent->cc_toflush = 0;
4696 _sd_centry_release(cc_ent);
4697 cc_ent = old_ent;
4698 categorize_centry = FOUND_IN_HASH_DM;
4699 } else {
4700 sdbc_centry_lost++;
4701
4702 CLEAR_CENTRY_INUSE(old_ent);
4703
4704 if (nowait) {
4705 _sd_centry_release(cc_ent);
4706 cc_ent = NULL;
4707 goto out;
4708 }
4709
4710 goto alloc_try;
4711 }
4712 }
4713
4714 SDTRACE(ST_EXIT|SDF_ENT_ALLOC, cd, tries,
4715 BLK_TO_FBA_NUM(cblk), 0, 0);
4716
4717 if (cc_ent->cc_await_use) {
4718 mutex_enter(&cc_ent->cc_lock);
4719 cv_broadcast(&cc_ent->cc_blkcv);
4720 mutex_exit(&cc_ent->cc_lock);
4721 }
4722
4723 sdbc_centry_init_dm(cc_ent);
4724
4725 cc_ent->cc_aging_dm |= categorize_centry;
4726
4727 out:
4728 return (cc_ent);
4729 }
4730
4731 SDTRACE(ST_INFO|SDF_ENT_ALLOC, cd, ++tries, BLK_TO_FBA_NUM(cblk), 0, 0);
4732
4733 delay(drv_usectohz(20000));
4734 (void) (*stall)++;
4735 num_tries = _sd_lruq_srch;
4736 goto retry_alloc_centry;
4737 }
4738
4739 /*
4740 * sdbc_centry_init_dm - setup the cache block for dynamic memory allocation
4741 *
4742 * ARGUMENTS:
4743 * centry - Cache block.
4744 *
4745 * RETURNS:
4746 * NONE
4747 *
4748 * USAGE:
4749 * This routine is the central point in which cache entry blocks are setup
4750 */
4751 static void
4752 sdbc_centry_init_dm(_sd_cctl_t *centry)
4753 {
4754
4755 /* an entry already setup - don't touch simply refresh age */
4756 if (centry->cc_data) {
4757 centry->cc_aging_dm &= ~(FINAL_AGING_DM);
4758
4759 DTRACE_PROBE1(sdbc_centry_init_dm_end,
4760 char *, centry->cc_data);
4761 return;
4762 }
4763
4764 centry->cc_aging_dm &= ~(FINAL_AGING_DM | CATAGORY_ENTRY_DM);
4765
4766 if (centry->cc_head_dm || centry->cc_next_dm)
4767 cmn_err(cmn_level, "!sdbc(sdbc_centry_init_dm): "
4768 "non-zero mem chain in ccent %p", (void *)centry);
4769
4770 centry->cc_head_dm = 0;
4771
4772 if (!sdbc_use_dmchain)
4773 centry->cc_next_dm = 0;
4774
4775 centry->cc_data = 0;
4776
4777 }
4778
4779 /*
4780 * sdbc_centry_memalloc_dm
4781 *
4782 * Actually allocate the cache memory, storing it in the cc_data field for
4783 * the cctl
4784 *
4785 * ARGS:
4786 * centry: cache control block for which to allocate the memory
4787 * alloc_request: number of bytes to allocate
4788 * flag: if called with ALLOC_NOWAIT, caller must check for non-zero return
4789 *
4790 * RETURNS:
4791 * 0 on success
4792 * non-zero on error
4793 */
4794 static int
4795 sdbc_centry_memalloc_dm(_sd_cctl_t *centry, int alloc_request, int flag)
4796 {
4797 int cblocks;
4798 _sd_queue_t *newq;
4799 int sleep;
4800 sleep = (flag & ALLOC_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
4801
4802 if (!centry->cc_data && (alloc_request > 0)) {
4803 /* host or other */
4804 dynmem_processing_dm.alloc_ct++;
4805 centry->cc_data = (unsigned char *)
4806 kmem_alloc((size_t)centry->cc_alloc_size_dm, sleep);
4807
4808
4809 if (sdbc_use_dmchain) {
4810 cblocks = centry->cc_alloc_size_dm >> _sd_cblock_shift;
4811 newq = &sdbc_dm_queues[cblocks];
4812
4813 /* set the dmqueue index */
4814 centry->cc_cblocks = cblocks;
4815
4816 /* put on appropriate queue */
4817 sdbc_ins_dmqueue_back(newq, centry);
4818 }
4819
4820 /*
4821 * for KM_NOSLEEP (should never happen with KM_SLEEP)
4822 */
4823 if (!centry->cc_data)
4824 return (LOW_RESOURCES_DM);
4825 centry->cc_head_dm = centry;
4826 centry->cc_alloc_ct_dm++;
4827 }
4828
4829 return (0);
4830 }
4831
4832 /*
4833 * _sd_centry_release - release a cache block
4834 *
4835 * ARGUMENTS:
4836 * centry - Cache block.
4837 *
4838 * RETURNS:
4839 * NONE
4840 *
4841 * USAGE:
4842 * This routine frees up a cache block. It also frees up a write
4843 * block if allocated and its valid to release it.
4844 */
4845
4846 void
4847 _sd_centry_release(_sd_cctl_t *centry)
4848 {
4849 ss_centry_info_t *wctl;
4850
4851 SDTRACE(ST_ENTER|SDF_ENT_FREE, CENTRY_CD(centry), 0,
4852 BLK_TO_FBA_NUM(CENTRY_BLK(centry)), 0, 0);
4853
4854 CLEAR_CENTRY_PAGEIO(centry);
4855
4856 if ((wctl = centry->cc_write) != 0) {
4857 /* was FAST */
4858 mutex_enter(¢ry->cc_lock);
4859 if (CENTRY_DIRTY(centry))
4860 wctl = NULL;
4861 else {
4862 centry->cc_write = NULL;
4863 centry->cc_flag &= ~(CC_PINNABLE);
4864 }
4865 /* was FAST */
4866 mutex_exit(¢ry->cc_lock);
4867 if (wctl) {
4868 wctl->sc_dirty = 0;
4869 SSOP_SETCENTRY(sdbc_safestore, wctl);
4870 SSOP_DEALLOCRESOURCE(sdbc_safestore, wctl->sc_res);
4871 }
4872 }
4873
4874 if (!(centry->cc_aging_dm & BAD_CHAIN_DM)) {
4875 if (sdbc_use_dmchain) {
4876 if (centry->cc_alloc_size_dm) {
4877
4878 /* see if this can be queued to head */
4879 if (CENTRY_QHEAD(centry)) {
4880 sdbc_requeue_head_dm_try(centry);
4881 } else {
4882 int qidx;
4883 _sd_queue_t *q;
4884
4885 qidx = centry->cc_cblocks;
4886 q = &sdbc_dm_queues[qidx];
4887
4888 if (_sd_lru_reinsert(q, centry)) {
4889 sdbc_requeue_dmchain(q,
4890 centry, 1, 1);
4891 }
4892 }
4893 } else {
4894 /*
4895 * Fix for bug 4949134:
4896 * If an internal block is marked with CC_QHEAD
4897 * but the HOST block is not, the chain will
4898 * never age properly, and will never be made
4899 * available. Only the HOST of the dmchain is
4900 * checked for CC_QHEAD, so clearing an internal
4901 * block indiscriminately (as is being done
4902 * here) does no damage.
4903 *
4904 * The same result could instead be achieved by
4905 * not setting the CC_QHEAD flag in the first
4906 * place, if the block is an internal dmchain
4907 * block, and if it is found in the hash table.
4908 * The current solution was chosen since it is
4909 * the least intrusive.
4910 */
4911 centry->cc_flag &= ~CC_QHEAD;
4912 }
4913 } else {
4914 if (CENTRY_QHEAD(centry)) {
4915 if (!CENTRY_DIRTY(centry))
4916 _sd_requeue_head(centry);
4917 } else if (_sd_lru_reinsert(_SD_LRU_Q, centry))
4918 _sd_requeue(centry);
4919 }
4920 }
4921
4922 SDTRACE(ST_EXIT|SDF_ENT_FREE, CENTRY_CD(centry), 0,
4923 BLK_TO_FBA_NUM(CENTRY_BLK(centry)), 0, 0);
4924
4925 /* only clear inuse after final reference to centry */
4926
4927 CLEAR_CENTRY_INUSE(centry);
4928 }
4929
4930
4931 /*
4932 * lookup to centry info associated with safestore resource
4933 * return pointer to the centry info structure
4934 */
4935 ss_centry_info_t *
4936 sdbc_get_cinfo_byres(ss_resource_t *res)
4937 {
4938 ss_centry_info_t *cinfo;
4939 ss_centry_info_t *cend;
4940 int found = 0;
4941
4942 ASSERT(res != NULL);
4943
4944 if (res == NULL)
4945 return (NULL);
4946
4947 cinfo = _sdbc_gl_centry_info;
4948 cend = _sdbc_gl_centry_info +
4949 (_sdbc_gl_centry_info_size / sizeof (ss_centry_info_t)) - 1;
4950
4951 for (; cinfo <= cend; ++cinfo)
4952 if (cinfo->sc_res == res) {
4953 ++found;
4954 break;
4955 }
4956
4957 if (!found)
4958 cinfo = NULL; /* bad */
4959
4960 return (cinfo);
4961 }
4962
4963 /*
4964 * _sd_alloc_write - Allocate a write block (for remote mirroring)
4965 * and set centry->cc_write
4966 *
4967 * ARGUMENTS:
4968 * centry - Head of Cache chain
4969 * stall - pointer to stall count (no blocks avail)
4970 *
4971 * RETURNS:
4972 * 0 - and sets cc_write for all entries when write contl block obtained.
4973 * -1 - if a write control block could not be obtained.
4974 */
4975
4976 int
4977 _sd_alloc_write(_sd_cctl_t *centry, int *stall)
4978 {
4979
4980 ss_resourcelist_t *reslist;
4981 ss_resourcelist_t *savereslist;
4982 ss_resource_t *res;
4983 _sd_cctl_t *ce;
4984 int err;
4985 int need;
4986
4987
4988 need = 0;
4989
4990 for (ce = centry; ce; ce = ce->cc_chain) {
4991 if (!(ce->cc_write))
4992 need++;
4993 }
4994
4995 if (!need)
4996 return (0);
4997
4998 if ((SSOP_ALLOCRESOURCE(sdbc_safestore, need, stall, &reslist))
4999 == SS_OK) {
5000 savereslist = reslist;
5001 for (ce = centry; ce; ce = ce->cc_chain) {
5002 if (ce->cc_write)
5003 continue;
5004 err = SSOP_GETRESOURCE(sdbc_safestore, &reslist, &res);
5005 if (err == SS_OK)
5006 ce->cc_write = sdbc_get_cinfo_byres(res);
5007
5008 ASSERT(err == SS_OK); /* panic if DEBUG on */
5009 ASSERT(ce->cc_write != NULL);
5010
5011 /*
5012 * this is bad and should not happen.
5013 * we use the saved reslist to cleanup
5014 * and return.
5015 */
5016 if ((err != SS_OK) || !ce->cc_write) {
5017
5018 cmn_err(CE_WARN, "!_sd_alloc_write: "
5019 "bad resource list 0x%p"
5020 "changing to forced write thru mode",
5021 (void *)savereslist);
5022
5023 (void) _sd_set_node_hint(NSC_FORCED_WRTHRU);
5024
5025 while (SSOP_GETRESOURCE(sdbc_safestore,
5026 &savereslist, &res) == SS_OK) {
5027
5028 SSOP_DEALLOCRESOURCE(sdbc_safestore,
5029 res);
5030 }
5031
5032 return (-1);
5033
5034 }
5035
5036 }
5037 return (0);
5038 }
5039
5040 /* no safestore resources available. do sync write */
5041 _sd_unblock(&_sd_flush_cv);
5042 return (-1);
5043 }
5044
5045 /*
5046 * _sd_read - Interface call to do read.
5047 *
5048 * ARGUMENTS:
5049 * handle - handle allocated earlier on.
5050 * fba_pos - disk block number to read from.
5051 * fba_len - length in fbas.
5052 * flag - flag: (NSC_NOBLOCK for async io)
5053 *
5054 * RETURNS:
5055 * errno if return > 0
5056 * NSC_DONE or NSC_PENDING otherwise.
5057 *
5058 * USAGE:
5059 * This routine checks if the request is valid and calls the underlying
5060 * doread routine (also called by alloc_buf)
5061 */
5062
5063 int
5064 _sd_read(_sd_buf_handle_t *handle, nsc_off_t fba_pos, nsc_size_t fba_len,
5065 int flag)
5066 {
5067 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
5068 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
5069 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
5070 _sd_cctl_t *cc_ent = NULL;
5071 nsc_size_t fba_orig_len = fba_len;
5072 int ret;
5073 int cd = HANDLE_CD(handle);
5074
5075 if (_sdbc_shutdown_in_progress || (handle->bh_flag & NSC_ABUF)) {
5076 ret = EIO;
5077 goto out;
5078 }
5079
5080
5081 #if !defined(_SD_NOCHECKS)
5082 if (!_SD_HANDLE_ACTIVE(handle)) {
5083 cmn_err(CE_WARN, "!sdbc(_sd_read) handle %p not active",
5084 (void *)handle);
5085 ret = EINVAL;
5086 goto out;
5087 }
5088 ASSERT_HANDLE_LIMITS(handle, fba_pos, fba_len);
5089 #endif
5090 if (fba_len == 0) {
5091 ret = NSC_DONE;
5092 goto out;
5093 }
5094
5095 KSTAT_RUNQ_ENTER(cd);
5096
5097 st_cblk_off = BLK_FBA_OFF(fba_pos);
5098 st_cblk_len = BLK_FBAS - st_cblk_off;
5099 if ((nsc_size_t)st_cblk_len >= fba_len) {
5100 end_cblk_len = 0;
5101 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
5102 } else {
5103 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
5104 }
5105
5106 cc_ent = handle->bh_centry;
5107 while (CENTRY_BLK(cc_ent) != FBA_TO_BLK_NUM(fba_pos))
5108 cc_ent = cc_ent->cc_chain;
5109
5110 if (!SDBC_VALID_BITS(st_cblk_off, st_cblk_len, cc_ent))
5111 goto need_io;
5112 DATA_LOG(SDF_RD, cc_ent, st_cblk_off, st_cblk_len);
5113
5114 DTRACE_PROBE4(_sd_read_data1, uint64_t,
5115 (uint64_t)(BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)) + st_cblk_off),
5116 uint64_t, (uint64_t)st_cblk_len, char *,
5117 *(int64_t *)(cc_ent->cc_data + FBA_SIZE(st_cblk_off)),
5118 char *, *(int64_t *)(cc_ent->cc_data +
5119 FBA_SIZE(st_cblk_off + st_cblk_len) - 8));
5120
5121 fba_pos += st_cblk_len;
5122 fba_len -= st_cblk_len;
5123 cc_ent = cc_ent->cc_chain;
5124
5125 while (fba_len > (nsc_size_t)end_cblk_len) {
5126 if (!FULLY_VALID(cc_ent))
5127 goto need_io;
5128 DATA_LOG(SDF_RD, cc_ent, 0, BLK_FBAS);
5129
5130 DTRACE_PROBE4(_sd_read_data2, uint64_t,
5131 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)),
5132 uint64_t, (uint64_t)BLK_FBAS,
5133 char *, *(int64_t *)(cc_ent->cc_data),
5134 char *, *(int64_t *)(cc_ent->cc_data +
5135 FBA_SIZE(BLK_FBAS) - 8));
5136
5137 fba_pos += BLK_FBAS;
5138 fba_len -= BLK_FBAS;
5139 cc_ent = cc_ent->cc_chain;
5140 }
5141 if (fba_len) {
5142 if (!SDBC_VALID_BITS(0, end_cblk_len, cc_ent))
5143 goto need_io;
5144 DATA_LOG(SDF_RD, cc_ent, 0, end_cblk_len);
5145
5146 DTRACE_PROBE4(_sd_read_data3, uint64_t,
5147 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)),
5148 uint64_t, (uint64_t)end_cblk_len,
5149 char *, *(int64_t *)(cc_ent->cc_data),
5150 char *, *(int64_t *)(cc_ent->cc_data +
5151 FBA_SIZE(end_cblk_len) - 8));
5152 }
5153
5154 CACHE_FBA_READ(handle->bh_cd, fba_orig_len);
5155 CACHE_READ_HIT;
5156
5157 FBA_READ_IO_KSTATS(handle->bh_cd, FBA_SIZE(fba_orig_len));
5158
5159 ret = NSC_HIT;
5160 goto stats_exit;
5161 need_io:
5162 _SD_DISCONNECT_CALLBACK(handle);
5163
5164 ret = _sd_doread(handle, cc_ent, fba_pos, fba_len, flag);
5165
5166 stats_exit:
5167 KSTAT_RUNQ_EXIT(cd);
5168 out:
5169 return (ret);
5170 }
5171
5172
5173 /*
5174 * sdbc_doread_prefetch - read ahead one cache block
5175 *
5176 * ARGUMENTS:
5177 * cc_ent - cache entry
5178 * fba_pos - disk block number to read from
5179 * fba_len - length in fbas.
5180 *
5181 * RETURNS:
5182 * number of fbas, if any, that are to be read beyond (fba_pos + fba_len)
5183 *
5184 * USAGE:
5185 * if readahead is to be done allocate a cache block and place
5186 * on the cc_chain of cc_ent
5187 */
5188 static int
5189 sdbc_doread_prefetch(_sd_cctl_t *cc_ent, nsc_off_t fba_pos, nsc_size_t fba_len)
5190 {
5191 nsc_off_t st_cblk = FBA_TO_BLK_NUM(fba_pos);
5192 nsc_off_t next_cblk = FBA_TO_BLK_NUM(fba_pos + BLK_FBAS);
5193 nsc_size_t filesize;
5194 int fba_count = 0; /* number of fbas to prefetch */
5195 _sd_cctl_t *cc_ra; /* the read ahead cache entry */
5196 int cd = CENTRY_CD(cc_ent);
5197 nsc_size_t vol_fill;
5198
5199 filesize = _sd_cache_files[cd].cd_info->sh_filesize;
5200 vol_fill = filesize - (fba_pos + fba_len);
5201
5202 /* readahead only for small reads */
5203 if ((fba_len <= FBA_LEN(CACHE_BLOCK_SIZE)) && (fba_pos != 0) &&
5204 (vol_fill > 0)) {
5205
5206 /*
5207 * if prev block is in cache and next block is not,
5208 * then read ahead one block
5209 */
5210 if (_sd_hash_search(cd, st_cblk - 1, _sd_htable)) {
5211 if (!_sd_hash_search(cd, next_cblk, _sd_htable)) {
5212
5213 cc_ra = sdbc_centry_alloc_blks
5214 (cd, next_cblk, 1, ALLOC_NOWAIT);
5215 if (cc_ra) {
5216 /* if in cache don't readahead */
5217 if (cc_ra->cc_aging_dm &
5218 HASH_ENTRY_DM) {
5219 ++sdbc_ra_hash;
5220 _sd_centry_release(cc_ra);
5221 } else {
5222 cc_ent->cc_chain = cc_ra;
5223 cc_ra->cc_chain = 0;
5224 fba_count =
5225 (vol_fill >
5226 (nsc_size_t)BLK_FBAS) ?
5227 BLK_FBAS : (int)vol_fill;
5228 /*
5229 * indicate implicit prefetch
5230 * and mark for release in
5231 * _sd_read_complete()
5232 */
5233 cc_ra->cc_aging_dm |=
5234 (PREFETCH_BUF_I |
5235 PREFETCH_BUF_IR);
5236 }
5237 } else {
5238 ++sdbc_ra_none;
5239 }
5240 }
5241 }
5242
5243 }
5244
5245 return (fba_count);
5246 }
5247
5248 /*
5249 * _sd_doread - Check if blocks in cache. If not completely true, do io.
5250 *
5251 * ARGUMENTS:
5252 * handle - handle allocated earlier on.
5253 * fba_pos - disk block number to read from.
5254 * fba_len - length in fbas.
5255 * flag - flag: (NSC_NOBLOCK for async io)
5256 *
5257 * RETURNS:
5258 * errno if return > 0
5259 * NSC_DONE(from disk), or NSC_PENDING otherwise.
5260 *
5261 * Comments:
5262 * It initiates an io and either blocks waiting for the completion
5263 * or return NSC_PENDING, depending on whether the flag bit
5264 * NSC_NOBLOCK is reset or set.
5265 *
5266 */
5267
5268
5269 static int
5270 _sd_doread(_sd_buf_handle_t *handle, _sd_cctl_t *cc_ent, nsc_off_t fba_pos,
5271 nsc_size_t fba_len, int flag)
5272 {
5273 int cd, err;
5274 nsc_size_t fba_orig_len; /* length in FBA's of the original request */
5275 nsc_size_t file_len; /* length in bytes of io to be done */
5276 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
5277 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
5278 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
5279 int num_bdl;
5280 _sd_cctl_t *cc_temp;
5281 struct buf *bp;
5282 unsigned int want_bits;
5283 void (*fn)(blind_t, nsc_off_t, nsc_size_t, int);
5284 sdbc_cblk_fba_t end_cblk_fill; /* FBA's to fill to end of last block */
5285 nsc_size_t vol_end_fill; /* # of FBA's to fill to end of the volume */
5286
5287 cd = HANDLE_CD(handle);
5288 SDTRACE(ST_ENTER|SDF_READ, cd, fba_len, fba_pos, flag, 0);
5289
5290 ASSERT(cd >= 0);
5291 if (_sd_cache_files[cd].cd_info->sh_failed) {
5292 SDTRACE(ST_EXIT|SDF_READ, cd, fba_len, fba_pos, flag, EIO);
5293 return (EIO);
5294 }
5295
5296 /*
5297 * adjust the position and length so that the entire cache
5298 * block is read in
5299 */
5300
5301 /* first, adjust to beginning of cache block */
5302
5303 fba_len += BLK_FBA_OFF(fba_pos); /* add start offset to length */
5304 fba_pos &= ~BLK_FBA_MASK; /* move position back to start of block */
5305
5306 /* compute fill to end of cache block */
5307 end_cblk_fill = (BLK_FBAS - 1) - ((fba_len - 1) % BLK_FBAS);
5308 vol_end_fill = _sd_cache_files[(cd)].cd_info->sh_filesize -
5309 (fba_pos + fba_len);
5310
5311 /* fill to lesser of cache block or end of volume */
5312 fba_len += ((nsc_size_t)end_cblk_fill < vol_end_fill) ? end_cblk_fill :
5313 vol_end_fill;
5314
5315 DTRACE_PROBE2(_sd_doread_rfill, nsc_off_t, fba_pos,
5316 nsc_size_t, fba_len);
5317
5318
5319 /* for small reads do 1-block readahead if previous block is in cache */
5320 if (sdbc_prefetch1)
5321 fba_len += sdbc_doread_prefetch(cc_ent, fba_pos, fba_len);
5322
5323 fba_orig_len = fba_len;
5324 st_cblk_off = BLK_FBA_OFF(fba_pos);
5325 st_cblk_len = BLK_FBAS - st_cblk_off;
5326 if ((nsc_size_t)st_cblk_len >= fba_len) {
5327 end_cblk_len = 0;
5328 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
5329 } else {
5330 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
5331 }
5332
5333 cc_temp = cc_ent;
5334 num_bdl = 0;
5335 while (cc_temp) {
5336 num_bdl += (SDBC_LOOKUP_IOCOUNT(CENTRY_DIRTY(cc_temp)));
5337 cc_temp = cc_temp->cc_chain;
5338 }
5339 bp = sd_alloc_iob(_sd_cache_files[cd].cd_crdev,
5340 fba_pos, num_bdl, B_READ);
5341 if (bp == NULL) {
5342 SDTRACE(ST_EXIT|SDF_READ, cd, fba_len, fba_pos, flag, E2BIG);
5343 return (E2BIG);
5344 }
5345
5346 want_bits = SDBC_GET_BITS(st_cblk_off, st_cblk_len);
5347 if (want_bits & CENTRY_DIRTY(cc_ent))
5348 _sd_ccent_rd(cc_ent, want_bits, bp);
5349 else {
5350 sd_add_fba(bp, &cc_ent->cc_addr, st_cblk_off, st_cblk_len);
5351 }
5352 file_len = FBA_SIZE(st_cblk_len);
5353 cc_ent = cc_ent->cc_chain;
5354 fba_len -= st_cblk_len;
5355
5356 while (fba_len > (nsc_size_t)end_cblk_len) {
5357 if (CENTRY_DIRTY(cc_ent))
5358 _sd_ccent_rd(cc_ent, (uint_t)BLK_FBA_BITS, bp);
5359 else {
5360 sd_add_fba(bp, &cc_ent->cc_addr, 0, BLK_FBAS);
5361 }
5362 file_len += CACHE_BLOCK_SIZE;
5363 cc_ent = cc_ent->cc_chain;
5364 fba_len -= BLK_FBAS;
5365 }
5366
5367 if (fba_len) {
5368 want_bits = SDBC_GET_BITS(0, end_cblk_len);
5369 if (want_bits & CENTRY_DIRTY(cc_ent))
5370 _sd_ccent_rd(cc_ent, want_bits, bp);
5371 else {
5372 sd_add_fba(bp, &cc_ent->cc_addr, 0, end_cblk_len);
5373 }
5374 file_len += FBA_SIZE(end_cblk_len);
5375 }
5376
5377 CACHE_READ_MISS;
5378 FBA_READ_IO_KSTATS(cd, file_len);
5379
5380 DISK_FBA_READ(cd, FBA_NUM(file_len));
5381
5382 fn = (handle->bh_flag & NSC_NOBLOCK) ? _sd_async_read_ea : NULL;
5383 err = sd_start_io(bp, _sd_cache_files[cd].cd_strategy, fn, handle);
5384
5385 if (err != NSC_PENDING) {
5386 _sd_read_complete(handle, fba_pos, fba_orig_len, err);
5387 }
5388
5389 SDTRACE(ST_EXIT|SDF_READ, cd, fba_orig_len, fba_pos, flag, err);
5390
5391 return (err);
5392 }
5393
5394
5395
5396 /*
5397 * _sd_read_complete - Do whatever is necessary after a read io is done.
5398 *
5399 * ARGUMENTS:
5400 * handle - handle allocated earlier on.
5401 * fba_pos - disk block number to read from.
5402 * fba_len - length in fbas.
5403 * error - error from io if any.
5404 *
5405 * RETURNS:
5406 * NONE.
5407 *
5408 * Comments:
5409 * This routine marks the cache blocks valid if the io completed
5410 * sucessfully. Called from the async end action as well as after
5411 * a synchrnous read completes.
5412 */
5413
5414 void
5415 _sd_read_complete(_sd_buf_handle_t *handle, nsc_off_t fba_pos,
5416 nsc_size_t fba_len, int error)
5417 {
5418 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
5419 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
5420 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
5421 nsc_size_t cur_fba_len; /* length in FBA's */
5422 _sd_cctl_t *cc_iocent;
5423 _sd_cctl_t *first_iocent; /* first buffer when processing prefetch */
5424
5425 cc_iocent = handle->bh_centry;
5426
5427 if ((handle->bh_error = error) == 0) {
5428 while (CENTRY_BLK(cc_iocent) != FBA_TO_BLK_NUM(fba_pos))
5429 cc_iocent = cc_iocent->cc_chain;
5430
5431 cur_fba_len = fba_len;
5432 st_cblk_off = BLK_FBA_OFF(fba_pos);
5433 st_cblk_len = BLK_FBAS - st_cblk_off;
5434 if ((nsc_size_t)st_cblk_len >= fba_len) {
5435 end_cblk_len = 0;
5436 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
5437 } else {
5438 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
5439 }
5440
5441 SDBC_SET_VALID_BITS(st_cblk_off, st_cblk_len, cc_iocent);
5442 DATA_LOG(SDF_RDIO, cc_iocent, st_cblk_off, st_cblk_len);
5443
5444 DTRACE_PROBE4(_sd_read_complete_data1, uint64_t, (uint64_t)
5445 BLK_TO_FBA_NUM(CENTRY_BLK(cc_iocent)) + st_cblk_off,
5446 int, st_cblk_len, char *,
5447 *(int64_t *)(cc_iocent->cc_data + FBA_SIZE(st_cblk_off)),
5448 char *, *(int64_t *)(cc_iocent->cc_data +
5449 FBA_SIZE(st_cblk_off + st_cblk_len) - 8));
5450
5451
5452 first_iocent = cc_iocent;
5453 cc_iocent = cc_iocent->cc_chain;
5454 cur_fba_len -= st_cblk_len;
5455
5456 while (cur_fba_len > (nsc_size_t)end_cblk_len) {
5457 SET_FULLY_VALID(cc_iocent);
5458 DATA_LOG(SDF_RDIO, cc_iocent, 0, BLK_FBAS);
5459
5460 DTRACE_PROBE4(_sd_read_complete_data2, uint64_t,
5461 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(cc_iocent)),
5462 int, BLK_FBAS, char *,
5463 *(int64_t *)(cc_iocent->cc_data), char *,
5464 *(int64_t *)(cc_iocent->cc_data +
5465 FBA_SIZE(BLK_FBAS) - 8));
5466
5467 /*
5468 * 4755485 release implicit prefetch buffers
5469 *
5470 * the cc_chain of the first buffer must NULL'd
5471 * else _sd_free_buf() will do a double free when
5472 * it traverses the chain.
5473 *
5474 * if a buffer has been marked PREFETCH_BUF_IR then
5475 * it is guaranteed that
5476 * 1. it is the second in a chain of two.
5477 * 2. cur_fba_len is BLK_FBAS.
5478 * 3. end_cblk_len is zero.
5479 *
5480 * because of 1 (and 2) above, we can safely exit the
5481 * while loop via the break statement without
5482 * executing the last two statements. the break
5483 * statement is necessary because it would be unsafe
5484 * to access cc_iocent which could be reallocated
5485 * immediately after the _sd_centry_release().
5486 */
5487 if (cc_iocent->cc_aging_dm & PREFETCH_BUF_IR) {
5488 cc_iocent->cc_aging_dm &= ~(PREFETCH_BUF_IR);
5489 _sd_centry_release(cc_iocent);
5490 first_iocent->cc_chain = NULL;
5491 break;
5492 }
5493
5494 cc_iocent = cc_iocent->cc_chain;
5495 cur_fba_len -= BLK_FBAS;
5496 }
5497 if (end_cblk_len) {
5498 SDBC_SET_VALID_BITS(0, end_cblk_len, cc_iocent);
5499 DATA_LOG(SDF_RDIO, cc_iocent, 0, end_cblk_len);
5500
5501 DTRACE_PROBE4(_sd_read_complete_data3, uint64_t,
5502 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(cc_iocent)),
5503 int, end_cblk_len, char *,
5504 *(int64_t *)(cc_iocent->cc_data), char *,
5505 *(int64_t *)(cc_iocent->cc_data +
5506 FBA_SIZE(end_cblk_len) - 8));
5507 }
5508 }
5509
5510 }
5511
5512
5513 /*
5514 * _sd_async_read_ea - End action for async reads.
5515 *
5516 * ARGUMENTS:
5517 * xhandle - handle allocated earlier on (cast to blind_t).
5518 * fba_pos - disk block number read from.
5519 * fba_len - length in fbas.
5520 * error - error from io if any.
5521 *
5522 * RETURNS:
5523 * NONE.
5524 *
5525 * Comments:
5526 * This routine is called at interrupt level when the io is done.
5527 * This is called only when read is asynchronous (NSC_NOBLOCK)
5528 */
5529
5530 static void
5531 _sd_async_read_ea(blind_t xhandle, nsc_off_t fba_pos, nsc_size_t fba_len,
5532 int error)
5533 {
5534 _sd_buf_handle_t *handle = xhandle;
5535 int cd;
5536
5537 if (error) {
5538 cd = HANDLE_CD(handle);
5539 ASSERT(cd >= 0);
5540 _sd_cache_files[cd].cd_info->sh_failed = 1;
5541 }
5542 SDTRACE(ST_ENTER|SDF_READ_EA, HANDLE_CD(handle),
5543 handle->bh_fba_len, handle->bh_fba_pos, 0, error);
5544
5545 _sd_read_complete(handle, fba_pos, fba_len, error);
5546
5547 #if defined(_SD_DEBUG_PATTERN)
5548 check_buf_consistency(handle, "rd");
5549 #endif
5550
5551 SDTRACE(ST_EXIT|SDF_READ_EA, HANDLE_CD(handle),
5552 handle->bh_fba_len, handle->bh_fba_pos, 0, 0);
5553 _SD_READ_CALLBACK(handle);
5554 }
5555
5556
5557 /*
5558 * _sd_async_write_ea - End action for async writes.
5559 *
5560 * ARGUMENTS:
5561 * xhandle - handle allocated earlier on. (cast to blind_t)
5562 * fba_pos - disk block number written to.
5563 * fba_len - length in fbas.
5564 * error - error from io if any.
5565 *
5566 * RETURNS:
5567 * NONE.
5568 *
5569 * Comments:
5570 * This routine is called at interrupt level when the write io is done.
5571 * This is called only when we are in write-through mode and the write
5572 * call indicated asynchronous callback. (NSC_NOBLOCK)
5573 */
5574
5575 /* ARGSUSED */
5576
5577 static void
5578 _sd_async_write_ea(blind_t xhandle, nsc_off_t fba_pos, nsc_size_t fba_len,
5579 int error)
5580 {
5581 _sd_buf_handle_t *handle = xhandle;
5582 handle->bh_error = error;
5583
5584 if (error)
5585 _sd_cache_files[HANDLE_CD(handle)].cd_info->sh_failed = 1;
5586
5587 _SD_WRITE_CALLBACK(handle);
5588 }
5589
5590 /*
5591 * update_dirty - set dirty bits in cache block which is already dirty
5592 * cc_inuse is held, need cc_lock to avoid race with _sd_process_pending
5593 * must check for I/O in-progress and set PEND_DIRTY.
5594 * return previous dirty bits
5595 * [if set _sd_process_pending will re-issue]
5596 */
5597 static _sd_bitmap_t
5598 update_dirty(_sd_cctl_t *cc_ent, sdbc_cblk_fba_t st_off, sdbc_cblk_fba_t st_len)
5599 {
5600 _sd_bitmap_t old;
5601
5602 /* was FAST */
5603 mutex_enter(&cc_ent->cc_lock);
5604 old = CENTRY_DIRTY(cc_ent);
5605 if (old) {
5606 /*
5607 * If we are writing to an FBA that is still marked dirty,
5608 * record a write cancellation.
5609 */
5610 if (old & SDBC_GET_BITS(st_off, st_len)) {
5611 CACHE_WRITE_CANCELLATION(CENTRY_CD(cc_ent));
5612 }
5613
5614 /* This is a write to a block that was already dirty */
5615 SDBC_SET_DIRTY(st_off, st_len, cc_ent);
5616 sd_serialize();
5617 if (CENTRY_IO_INPROGRESS(cc_ent))
5618 cc_ent->cc_flag |= CC_PEND_DIRTY;
5619 }
5620 /* was FAST */
5621 mutex_exit(&cc_ent->cc_lock);
5622 return (old);
5623 }
5624
5625 /*
5626 * _sd_write - Interface call to commit part of handle.
5627 *
5628 * ARGUMENTS:
5629 * handle - handle allocated earlier o.
5630 * fba_pos - disk block number to write to.
5631 * fba_len - length in fbas.
5632 * flag - (NSC_NOBLOCK | NSC_WRTHRU)
5633 *
5634 * RETURNS:
5635 * errno if return > 0
5636 * NSC_HIT (in cache), NSC_DONE (to disk) or NSC_PENDING otherwise.
5637 *
5638 * Comments:
5639 * This routine checks validity of the handle and then calls the
5640 * sync-write function if this write is determined to be write-through.
5641 * Else, it reflects the data to the write blocks on the mirror node,
5642 * (allocated in alloc_buf). If the cache block is not dirty, it is
5643 * marked dirty and queued up for io processing later on.
5644 * If parts are already dirty but io is not in progress yet, it is
5645 * marked dirty and left alone (it is already in the queue)
5646 * If parts are already dirty but io is in progress, it is marked
5647 * dirty and also a flag is set indicating that this buffer should
5648 * be reprocessed after the io-end-action.
5649 * Attempt is made to coalesce multiple writes into a single list
5650 * for io processing later on.
5651 *
5652 * Issuing of writes may be delayed until the handle is released;
5653 * _sd_queue_write() sets NSC_QUEUE, indicating that dirty bits
5654 * and reflection to mirror have already been done, just queue I/O.
5655 */
5656
5657
5658
5659 int
5660 _sd_write(_sd_buf_handle_t *handle, nsc_off_t fba_pos, nsc_size_t fba_len,
5661 int flag)
5662 {
5663 int cd = HANDLE_CD(handle);
5664 int num_queued, ret, queue_only, store_only;
5665 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
5666 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
5667 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
5668 nsc_size_t cur_fba_len; /* position in disk blocks */
5669 _sd_cctl_t *cc_ent = NULL;
5670 _sd_cctl_t *cur_chain = NULL, *dirty_next = NULL;
5671
5672
5673 if (_sdbc_shutdown_in_progress) {
5674 ret = EIO;
5675 goto out;
5676 }
5677
5678
5679 if (!_SD_HANDLE_ACTIVE(handle)) {
5680 SDALERT(SDF_WRITE,
5681 SDT_INV_CD, 0, SDT_INV_BL, handle->bh_flag, 0);
5682 ret = EINVAL;
5683 goto out;
5684 }
5685 #if !defined(_SD_NOCHECKS)
5686 ASSERT_HANDLE_LIMITS(handle, fba_pos, fba_len);
5687 if ((handle->bh_flag & NSC_WRBUF) == 0) {
5688 ret = EINVAL;
5689 goto out;
5690 }
5691 #endif
5692 if (fba_len == 0) {
5693 ret = NSC_DONE;
5694 goto out;
5695 }
5696
5697 /*
5698 * store_only: don't queue this I/O yet
5699 * queue_only: queue I/O to disk, don't store in mirror node
5700 */
5701 if (flag & NSC_QUEUE)
5702 queue_only = 1, store_only = 0;
5703 else
5704 if (_SD_DELAY_QUEUE && (fba_len != handle->bh_fba_len))
5705 queue_only = 0, store_only = 1;
5706 else
5707 queue_only = store_only = 0;
5708
5709 if (!queue_only && _SD_FORCE_DISCONNECT(fba_len))
5710 _SD_DISCONNECT_CALLBACK(handle);
5711
5712 if (_sd_cache_files[cd].cd_info->sh_failed) {
5713 ret = EIO;
5714 goto out;
5715 }
5716
5717 KSTAT_RUNQ_ENTER(cd);
5718
5719 SDTRACE(ST_ENTER|SDF_WRITE, cd, fba_len, fba_pos, flag, 0);
5720
5721 #if defined(_SD_DEBUG_PATTERN)
5722 check_buf_consistency(handle, "wr");
5723 #endif
5724
5725 cc_ent = handle->bh_centry;
5726
5727 while (CENTRY_BLK(cc_ent) != FBA_TO_BLK_NUM(fba_pos))
5728 cc_ent = cc_ent->cc_chain;
5729
5730 if (((handle->bh_flag | flag) & _SD_WRTHRU_MASK) ||
5731 (!queue_only && _sd_remote_store(cc_ent, fba_pos, fba_len))) {
5732 flag |= NSC_WRTHRU;
5733
5734 ret = _sd_sync_write(handle, fba_pos, fba_len, flag);
5735 goto stats_exit;
5736 }
5737
5738 if (store_only) /* enqueue in _sd_free_buf() */
5739 handle->bh_flag |= NSC_QUEUE;
5740 cur_fba_len = fba_len;
5741 st_cblk_off = BLK_FBA_OFF(fba_pos);
5742 st_cblk_len = BLK_FBAS - st_cblk_off;
5743 if ((nsc_size_t)st_cblk_len >= fba_len) {
5744 end_cblk_len = 0;
5745 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
5746 } else {
5747 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
5748 }
5749
5750 if (CENTRY_DIRTY(cc_ent) && update_dirty(cc_ent, st_cblk_off,
5751 st_cblk_len))
5752 goto loop1;
5753 if (store_only) {
5754 SDBC_SET_TOFLUSH(st_cblk_off, st_cblk_len, cc_ent);
5755 goto loop1;
5756 }
5757 SDBC_SET_DIRTY(st_cblk_off, st_cblk_len, cc_ent);
5758 cur_chain = dirty_next = cc_ent;
5759 num_queued = 1;
5760
5761 loop1:
5762 DATA_LOG(SDF_WR, cc_ent, st_cblk_off, st_cblk_len);
5763
5764 DTRACE_PROBE4(_sd_write_data1, uint64_t, (uint64_t)
5765 (BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)) + st_cblk_off),
5766 int, st_cblk_len, char *,
5767 *(int64_t *)(cc_ent->cc_data + FBA_SIZE(st_cblk_off)),
5768 char *, *(int64_t *)(cc_ent->cc_data +
5769 FBA_SIZE(st_cblk_off+ st_cblk_len) - 8));
5770
5771 cur_fba_len -= st_cblk_len;
5772 cc_ent = cc_ent->cc_chain;
5773
5774 while (cur_fba_len > (nsc_size_t)end_cblk_len) {
5775 if (CENTRY_DIRTY(cc_ent) && update_dirty(cc_ent, 0, BLK_FBAS)) {
5776 if (cur_chain) {
5777 _sd_enqueue_dirty(cd, cur_chain, dirty_next,
5778 num_queued);
5779 cur_chain = dirty_next = NULL;
5780 }
5781 goto loop2;
5782 }
5783 if (store_only) {
5784 SDBC_SET_TOFLUSH(0, BLK_FBAS, cc_ent);
5785 goto loop2;
5786 }
5787 SDBC_SET_DIRTY(0, BLK_FBAS, cc_ent);
5788 if (dirty_next) {
5789 dirty_next->cc_dirty_next = cc_ent;
5790 dirty_next = cc_ent;
5791 num_queued++;
5792 } else {
5793 cur_chain = dirty_next = cc_ent;
5794 num_queued = 1;
5795 }
5796 loop2:
5797 DATA_LOG(SDF_WR, cc_ent, 0, BLK_FBAS);
5798
5799 DTRACE_PROBE4(_sd_write_data2, uint64_t,
5800 (uint64_t)(BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent))),
5801 int, BLK_FBAS, char *, *(int64_t *)(cc_ent->cc_data),
5802 char *, *(int64_t *)(cc_ent->cc_data +
5803 FBA_SIZE(BLK_FBAS) - 8));
5804
5805 cc_ent = cc_ent->cc_chain;
5806 cur_fba_len -= BLK_FBAS;
5807 }
5808
5809 #if defined(_SD_DEBUG)
5810 if (cur_fba_len != end_cblk_len)
5811 cmn_err(CE_WARN, "!fba_len %" NSC_SZFMT " end_cblk_len %d in "
5812 "_sd_write", cur_fba_len, end_cblk_len);
5813 #endif
5814
5815 if (cur_fba_len) {
5816 if (CENTRY_DIRTY(cc_ent) && update_dirty(cc_ent, 0,
5817 end_cblk_len)) {
5818 if (cur_chain) {
5819 _sd_enqueue_dirty(cd, cur_chain, dirty_next,
5820 num_queued);
5821 cur_chain = dirty_next = NULL;
5822 }
5823 goto loop3;
5824 }
5825 if (store_only) {
5826 SDBC_SET_TOFLUSH(0, end_cblk_len, cc_ent);
5827 goto loop3;
5828 }
5829 SDBC_SET_DIRTY(0, end_cblk_len, cc_ent);
5830 if (dirty_next) {
5831 dirty_next->cc_dirty_next = cc_ent;
5832 dirty_next = cc_ent;
5833 num_queued++;
5834 } else {
5835 cur_chain = dirty_next = cc_ent;
5836 num_queued = 1;
5837 }
5838 }
5839 loop3:
5840 if (cur_fba_len) {
5841 DATA_LOG(SDF_WR, cc_ent, 0, end_cblk_len);
5842
5843 DTRACE_PROBE4(_sd_write_data3, uint64_t,
5844 (uint64_t)(BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent))),
5845 int, end_cblk_len, char *, *(int64_t *)(cc_ent->cc_data),
5846 char *, *(int64_t *)(cc_ent->cc_data +
5847 FBA_SIZE(end_cblk_len) - 8));
5848
5849 }
5850
5851 if (!store_only && cur_chain) {
5852 _sd_enqueue_dirty(cd, cur_chain, dirty_next, num_queued);
5853 }
5854
5855 if (!queue_only) {
5856 CACHE_FBA_WRITE(cd, fba_len);
5857 CACHE_WRITE_HIT;
5858
5859 FBA_WRITE_IO_KSTATS(cd, FBA_SIZE(fba_len));
5860 }
5861
5862 ret = NSC_HIT;
5863
5864 stats_exit:
5865 SDTRACE(ST_EXIT|SDF_WRITE, cd, fba_len, fba_pos, flag, ret);
5866 KSTAT_RUNQ_EXIT(cd);
5867 out:
5868 return (ret);
5869 }
5870
5871
5872 /*
5873 * _sd_queue_write(handle, fba_pos, fba_len): Queues delayed writes for
5874 * flushing
5875 *
5876 * ARGUMENTS: handle - handle allocated with NSC_WRBUF
5877 * fba_pos - starting fba pos from _sd_alloc_buf()
5878 * fba_len - fba len from _sd_alloc_buf()
5879 *
5880 * USAGE : Called if _SD_DELAY_QUEUE is set. Finds all blocks in the
5881 * handle marked for flushing and queues them to be written in
5882 * optimized (i.e. sequential) order
5883 */
5884 static void
5885 _sd_queue_write(_sd_buf_handle_t *handle, nsc_off_t fba_pos, nsc_size_t fba_len)
5886 {
5887 nsc_off_t fba_end;
5888 sdbc_cblk_fba_t sblk, len, dirty;
5889 _sd_cctl_t *cc_ent;
5890 nsc_off_t flush_pos;
5891 int flush_pos_valid = 0;
5892 nsc_size_t flush_len = 0;
5893
5894 cc_ent = handle->bh_centry;
5895 fba_end = fba_pos + fba_len;
5896 fba_pos = BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)); /* 1st block */
5897 while (fba_pos < fba_end) {
5898 dirty = cc_ent->cc_toflush;
5899 cc_ent->cc_toflush = 0;
5900 /*
5901 * Full block
5902 */
5903 if (_SD_BMAP_ISFULL(dirty)) {
5904 if (flush_pos_valid == 0) {
5905 flush_pos_valid = 1;
5906 flush_pos = fba_pos;
5907 }
5908 flush_len += BLK_FBAS;
5909 }
5910 /*
5911 * Partial block
5912 */
5913 else while (dirty) {
5914 sblk = SDBC_LOOKUP_STPOS(dirty);
5915 len = SDBC_LOOKUP_LEN(dirty);
5916 SDBC_LOOKUP_MODIFY(dirty);
5917
5918 if (sblk && flush_pos_valid) {
5919 (void) _sd_write(handle, flush_pos, flush_len,
5920 NSC_QUEUE);
5921 flush_pos_valid = 0;
5922 flush_len = 0;
5923 }
5924 if (flush_pos_valid == 0) {
5925 flush_pos_valid = 1;
5926 flush_pos = fba_pos + sblk;
5927 }
5928 flush_len += len;
5929 }
5930 fba_pos += BLK_FBAS;
5931 cc_ent = cc_ent->cc_chain;
5932 /*
5933 * If we find a gap, write out what we've got
5934 */
5935 if (flush_pos_valid && (flush_pos + flush_len) != fba_pos) {
5936 (void) _sd_write(handle, flush_pos, flush_len,
5937 NSC_QUEUE);
5938 flush_pos_valid = 0;
5939 flush_len = 0;
5940 }
5941 }
5942 if (flush_pos_valid)
5943 (void) _sd_write(handle, flush_pos, flush_len, NSC_QUEUE);
5944 }
5945
5946
5947 static int
5948 _sd_remote_store(_sd_cctl_t *cc_ent, nsc_off_t fba_pos, nsc_size_t fba_len)
5949 {
5950 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
5951 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
5952 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
5953 ss_resource_t *ss_res;
5954
5955 if (_sd_nodes_configured <= 2 && _sd_is_mirror_down())
5956 return (0);
5957 st_cblk_off = BLK_FBA_OFF(fba_pos);
5958 st_cblk_len = BLK_FBAS - st_cblk_off;
5959 if ((nsc_size_t)st_cblk_len >= fba_len) {
5960 end_cblk_len = 0;
5961 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
5962 } else {
5963 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
5964 }
5965
5966 fba_len -= st_cblk_len;
5967
5968 ss_res = cc_ent->cc_write->sc_res;
5969 if (SSOP_WRITE_CBLOCK(sdbc_safestore, ss_res,
5970 cc_ent->cc_data + FBA_SIZE(st_cblk_off), FBA_SIZE(st_cblk_len),
5971 FBA_SIZE(st_cblk_off))) {
5972
5973 cmn_err(CE_WARN,
5974 "!sdbc(_sd_write) safe store failed. Going synchronous");
5975 SDTRACE(SDF_REFLECT, CENTRY_CD(cc_ent), fba_len,
5976 fba_pos, 0, -1);
5977 return (-1);
5978 }
5979
5980 cc_ent = cc_ent->cc_chain;
5981 while (fba_len > (nsc_size_t)end_cblk_len) {
5982 fba_len -= BLK_FBAS;
5983
5984 if (SSOP_WRITE_CBLOCK(sdbc_safestore, ss_res, cc_ent->cc_data,
5985 CACHE_BLOCK_SIZE, 0)) {
5986
5987 cmn_err(CE_WARN, "!sdbc(_sd_write) safe store failed. "
5988 "Going synchronous");
5989 SDTRACE(SDF_REFLECT, CENTRY_CD(cc_ent), fba_len,
5990 fba_pos, 0, -1);
5991 return (-1);
5992 }
5993
5994 cc_ent = cc_ent->cc_chain;
5995 } /* end while */
5996
5997 if (fba_len) {
5998 if (SSOP_WRITE_CBLOCK(sdbc_safestore, ss_res,
5999 cc_ent->cc_data, FBA_SIZE(end_cblk_len), 0)) {
6000
6001 cmn_err(CE_WARN, "!sdbc(_sd_write) nvmem dma failed. "
6002 "Going synchronous");
6003 SDTRACE(SDF_REFLECT, CENTRY_CD(cc_ent), fba_len,
6004 fba_pos, 0, -1);
6005 return (-1);
6006 }
6007 }
6008 return (0);
6009 }
6010
6011
6012 /*
6013 * _sd_sync_write2 - Write-through function.
6014 *
6015 * ARGUMENTS:
6016 * wr_handle - handle into which to write the data.
6017 * wr_st_pos - starting FBA position in wr_handle.
6018 * fba_len - length in fbas.
6019 * flag - NSC_NOBLOCK for async io.
6020 * rd_handle - handle from which to read the data, or NULL.
6021 * rd_st_pos - starting FBA position in rd_handle.
6022 *
6023 * RETURNS:
6024 * errno if return > 0
6025 * NSC_DONE or NSC_PENDING otherwise.
6026 *
6027 * Comments:
6028 * This routine initiates io of the indicated portion. It returns
6029 * synchronously after io is completed if NSC_NOBLOCK is not set.
6030 * Else NSC_PENDING is returned with a subsequent write callback on
6031 * io completion.
6032 *
6033 * See _sd_copy_direct() for usage when
6034 * (wr_handle != rd_handle && rd_handle != NULL)
6035 */
6036
6037 static int
6038 _sd_sync_write2(_sd_buf_handle_t *wr_handle, nsc_off_t wr_st_pos,
6039 nsc_size_t fba_len, int flag, _sd_buf_handle_t *rd_handle,
6040 nsc_off_t rd_st_pos)
6041 {
6042 void (*fn)(blind_t, nsc_off_t, nsc_size_t, int);
6043 _sd_cctl_t *wr_ent, *rd_ent;
6044 nsc_size_t this_len;
6045 nsc_off_t rd_pos, wr_pos;
6046 nsc_size_t log_bytes;
6047 int cd = HANDLE_CD(wr_handle);
6048 int err;
6049 uint_t dirty;
6050 struct buf *bp;
6051
6052 LINTUSED(flag);
6053
6054 _SD_DISCONNECT_CALLBACK(wr_handle);
6055
6056 if (rd_handle == NULL) {
6057 rd_handle = wr_handle;
6058 rd_st_pos = wr_st_pos;
6059 }
6060
6061 wr_ent = wr_handle->bh_centry;
6062 while (CENTRY_BLK(wr_ent) != FBA_TO_BLK_NUM(wr_st_pos))
6063 wr_ent = wr_ent->cc_chain;
6064
6065 rd_ent = rd_handle->bh_centry;
6066 while (CENTRY_BLK(rd_ent) != FBA_TO_BLK_NUM(rd_st_pos))
6067 rd_ent = rd_ent->cc_chain;
6068
6069 bp = sd_alloc_iob(_sd_cache_files[cd].cd_crdev,
6070 wr_st_pos, FBA_TO_BLK_LEN(fba_len) + 2, B_WRITE);
6071
6072 if (bp == NULL)
6073 return (E2BIG);
6074
6075 wr_pos = BLK_FBA_OFF(wr_st_pos);
6076 rd_pos = BLK_FBA_OFF(rd_st_pos);
6077 log_bytes = 0;
6078
6079 do {
6080 this_len = min((BLK_FBAS - rd_pos), (BLK_FBAS - wr_pos));
6081
6082 if (this_len > fba_len)
6083 this_len = fba_len;
6084
6085 /*
6086 * clear dirty bits in the write handle.
6087 */
6088
6089 if (CENTRY_DIRTY(wr_ent)) {
6090 mutex_enter(&wr_ent->cc_lock);
6091
6092 if (CENTRY_DIRTY(wr_ent)) {
6093 if (this_len == (nsc_size_t)BLK_FBAS ||
6094 rd_handle != wr_handle) {
6095 /*
6096 * optimization for when we have a
6097 * full cache block, or are doing
6098 * copy_direct (see below).
6099 */
6100
6101 wr_ent->cc_write->sc_dirty = 0;
6102 } else {
6103 dirty = wr_ent->cc_write->sc_dirty;
6104 dirty &= ~(SDBC_GET_BITS(
6105 wr_pos, this_len));
6106 wr_ent->cc_write->sc_dirty = dirty;
6107 }
6108
6109 SSOP_SETCENTRY(sdbc_safestore,
6110 wr_ent->cc_write);
6111 }
6112
6113 mutex_exit(&wr_ent->cc_lock);
6114 }
6115
6116 /*
6117 * update valid bits in the write handle.
6118 */
6119
6120 if (rd_handle == wr_handle) {
6121 if (this_len == (nsc_size_t)BLK_FBAS) {
6122 SET_FULLY_VALID(wr_ent);
6123 } else {
6124 SDBC_SET_VALID_BITS(wr_pos, this_len, wr_ent);
6125 }
6126 } else {
6127 /*
6128 * doing copy_direct, so mark the write handle
6129 * as invalid since the data is on disk, but not
6130 * in cache.
6131 */
6132 wr_ent->cc_valid = 0;
6133 }
6134
6135 DATA_LOG(SDF_WRSYNC, rd_ent, rd_pos, this_len);
6136
6137 DTRACE_PROBE4(_sd_sync_write2_data, uint64_t,
6138 (uint64_t)BLK_TO_FBA_NUM(CENTRY_BLK(rd_ent)) + rd_pos,
6139 uint64_t, (uint64_t)this_len, char *,
6140 *(int64_t *)(rd_ent->cc_data + FBA_SIZE(rd_pos)),
6141 char *, *(int64_t *)(rd_ent->cc_data +
6142 FBA_SIZE(rd_pos + this_len) - 8));
6143
6144 sd_add_fba(bp, &rd_ent->cc_addr, rd_pos, this_len);
6145
6146 log_bytes += FBA_SIZE(this_len);
6147 fba_len -= this_len;
6148
6149 wr_pos += this_len;
6150 if (wr_pos >= (nsc_size_t)BLK_FBAS) {
6151 wr_ent = wr_ent->cc_chain;
6152 wr_pos = 0;
6153 }
6154
6155 rd_pos += this_len;
6156 if (rd_pos >= (nsc_size_t)BLK_FBAS) {
6157 rd_ent = rd_ent->cc_chain;
6158 rd_pos = 0;
6159 }
6160
6161 } while (fba_len > 0);
6162
6163 DISK_FBA_WRITE(cd, FBA_NUM(log_bytes));
6164 CACHE_WRITE_MISS;
6165
6166 FBA_WRITE_IO_KSTATS(cd, log_bytes);
6167
6168 fn = (wr_handle->bh_flag & NSC_NOBLOCK) ? _sd_async_write_ea : NULL;
6169
6170 err = sd_start_io(bp, _sd_cache_files[cd].cd_strategy, fn, wr_handle);
6171
6172 if (err != NSC_PENDING) {
6173 DATA_LOG_CHAIN(SDF_WRSYEA, wr_handle->bh_centry,
6174 wr_st_pos, FBA_NUM(log_bytes));
6175 }
6176
6177 return (err);
6178 }
6179
6180
6181 static int
6182 _sd_sync_write(_sd_buf_handle_t *handle, nsc_off_t fba_pos, nsc_size_t fba_len,
6183 int flag)
6184 {
6185 return (_sd_sync_write2(handle, fba_pos, fba_len, flag, NULL, 0));
6186 }
6187
6188
6189 /*
6190 * _sd_zero - Interface call to zero out a portion of cache blocks.
6191 *
6192 * ARGUMENTS:
6193 * handle - handle allocated earlier on.
6194 * fba_pos - disk block number to zero from.
6195 * fba_len - length in fbas.
6196 * flag - NSC_NOBLOCK for async io.
6197 *
6198 * RETURNS:
6199 * errno if return > 0
6200 * NSC_DONE or NSC_PENDING otherwise.
6201 *
6202 * Comments:
6203 * This routine zeroes out the indicated portion of the cache blocks
6204 * and commits the data to disk.
6205 * (See write for more details on the commit)
6206 */
6207
6208
6209 int
6210 _sd_zero(_sd_buf_handle_t *handle, nsc_off_t fba_pos, nsc_size_t fba_len,
6211 int flag)
6212 {
6213 int cd;
6214 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
6215 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
6216 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
6217 nsc_size_t cur_fba_len; /* position in disk blocks */
6218 int ret;
6219 _sd_cctl_t *cc_ent;
6220
6221 if (_sdbc_shutdown_in_progress) {
6222 DTRACE_PROBE(shutdown);
6223 return (EIO);
6224 }
6225
6226 if (!_SD_HANDLE_ACTIVE(handle)) {
6227 cmn_err(CE_WARN, "!sdbc(_sd_zero) handle %p not active",
6228 (void *)handle);
6229
6230 DTRACE_PROBE1(handle_active, int, handle->bh_flag);
6231
6232 return (EINVAL);
6233 }
6234 ASSERT_HANDLE_LIMITS(handle, fba_pos, fba_len);
6235 if ((handle->bh_flag & NSC_WRBUF) == 0) {
6236 DTRACE_PROBE1(handle_write, int, handle->bh_flag);
6237 return (EINVAL);
6238 }
6239
6240 if (fba_len == 0) {
6241 DTRACE_PROBE(zero_len);
6242 return (NSC_DONE);
6243 }
6244
6245 if (_SD_FORCE_DISCONNECT(fba_len))
6246 _SD_DISCONNECT_CALLBACK(handle);
6247
6248 cd = HANDLE_CD(handle);
6249 SDTRACE(ST_ENTER|SDF_ZERO, cd, fba_len, fba_pos, flag, 0);
6250
6251 cc_ent = handle->bh_centry;
6252 while (CENTRY_BLK(cc_ent) != FBA_TO_BLK_NUM(fba_pos))
6253 cc_ent = cc_ent->cc_chain;
6254 cur_fba_len = fba_len;
6255 st_cblk_off = BLK_FBA_OFF(fba_pos);
6256 st_cblk_len = BLK_FBAS - st_cblk_off;
6257 if ((nsc_size_t)st_cblk_len >= fba_len) {
6258 end_cblk_len = 0;
6259 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
6260 } else {
6261 end_cblk_len = BLK_FBA_OFF(fba_pos + fba_len);
6262 }
6263
6264 cur_fba_len -= st_cblk_len;
6265 bzero(cc_ent->cc_data + FBA_SIZE(st_cblk_off), FBA_SIZE(st_cblk_len));
6266
6267 cc_ent = cc_ent->cc_chain;
6268 while (cur_fba_len > (nsc_size_t)end_cblk_len) {
6269 cur_fba_len -= BLK_FBAS;
6270 bzero(cc_ent->cc_data, CACHE_BLOCK_SIZE);
6271 cc_ent = cc_ent->cc_chain;
6272 }
6273 if (cur_fba_len) {
6274 bzero(cc_ent->cc_data, FBA_SIZE(cur_fba_len));
6275 }
6276
6277 ret = _sd_write(handle, fba_pos, fba_len, flag);
6278 SDTRACE(ST_EXIT|SDF_ZERO, cd, fba_len, fba_pos, flag, ret);
6279
6280 return (ret);
6281 }
6282
6283
6284 /*
6285 * _sd_copy - Copies portions of 2 handles.
6286 *
6287 * ARGUMENTS:
6288 * handle1 - handle allocated earlier on.
6289 * handle2 - handle allocated earlier on.
6290 * fba_pos1 - disk block number to read from.
6291 * fba_pos2 - disk block number to write to.
6292 * fba_len - length in fbas.
6293 *
6294 * RETURNS:
6295 * errno if return > 0
6296 * NSC_DONE otherwise.
6297 *
6298 * Comments:
6299 * This routine copies the 2 handles.
6300 * WARNING: this could put the cache blocks in the destination handle
6301 * in an inconsistent state. (the blocks could be valid in cache,
6302 * but the copy makes the cache different from disk)
6303 *
6304 */
6305
6306
6307 int
6308 _sd_copy(_sd_buf_handle_t *handle1, _sd_buf_handle_t *handle2,
6309 nsc_off_t fba_pos1, nsc_off_t fba_pos2, nsc_size_t fba_len)
6310 {
6311 sdbc_cblk_fba_t st_cblk_len; /* FBA len of starting cache block */
6312 sdbc_cblk_fba_t end_cblk_len; /* FBA len of ending cache block */
6313 sdbc_cblk_fba_t st_cblk_off; /* FBA offset into starting cblock */
6314 nsc_off_t off1, off2; /* offsets in FBA's into the disk */
6315 nsc_size_t cur_fba_len; /* position in disk blocks */
6316 _sd_cctl_t *cc_ent1, *cc_ent2;
6317
6318 if (_sdbc_shutdown_in_progress) {
6319 DTRACE_PROBE(shutdown);
6320 return (EIO);
6321 }
6322 if (!_SD_HANDLE_ACTIVE(handle1) || !_SD_HANDLE_ACTIVE(handle2)) {
6323 cmn_err(CE_WARN, "!sdbc(_sd_copy) handle %p or %p not active",
6324 (void *)handle1, (void *)handle2);
6325
6326 DTRACE_PROBE2(handle_active1, int, handle1->bh_flag,
6327 int, handle2->bh_flag);
6328
6329 return (EINVAL);
6330 }
6331 ASSERT_HANDLE_LIMITS(handle1, fba_pos1, fba_len);
6332 ASSERT_HANDLE_LIMITS(handle2, fba_pos2, fba_len);
6333
6334 cc_ent1 = handle1->bh_centry;
6335 while (CENTRY_BLK(cc_ent1) != FBA_TO_BLK_NUM(fba_pos1))
6336 cc_ent1 = cc_ent1->cc_chain;
6337
6338 cc_ent2 = handle2->bh_centry;
6339 while (CENTRY_BLK(cc_ent2) != FBA_TO_BLK_NUM(fba_pos2))
6340 cc_ent2 = cc_ent2->cc_chain;
6341
6342 if (BLK_FBA_OFF(fba_pos1) != BLK_FBA_OFF(fba_pos2)) {
6343 /* Different offsets, do it slowly (per fba) */
6344
6345 while (fba_len) {
6346 off1 = FBA_SIZE(BLK_FBA_OFF(fba_pos1));
6347 off2 = FBA_SIZE(BLK_FBA_OFF(fba_pos2));
6348
6349 bcopy(cc_ent1->cc_data+off1, cc_ent2->cc_data+off2,
6350 FBA_SIZE(1));
6351
6352 fba_pos1++;
6353 fba_pos2++;
6354 fba_len--;
6355
6356 if (FBA_TO_BLK_NUM(fba_pos1) != CENTRY_BLK(cc_ent1))
6357 cc_ent1 = cc_ent1->cc_chain;
6358 if (FBA_TO_BLK_NUM(fba_pos2) != CENTRY_BLK(cc_ent2))
6359 cc_ent2 = cc_ent2->cc_chain;
6360 }
6361
6362 DTRACE_PROBE(_sd_copy_end);
6363 return (NSC_DONE);
6364 }
6365 cur_fba_len = fba_len;
6366 st_cblk_off = BLK_FBA_OFF(fba_pos1);
6367 st_cblk_len = BLK_FBAS - st_cblk_off;
6368 if ((nsc_size_t)st_cblk_len >= fba_len) {
6369 end_cblk_len = 0;
6370 st_cblk_len = (sdbc_cblk_fba_t)fba_len;
6371 } else {
6372 end_cblk_len = BLK_FBA_OFF(fba_pos1 + fba_len);
6373 }
6374
6375 bcopy(cc_ent1->cc_data + FBA_SIZE(st_cblk_off),
6376 cc_ent2->cc_data + FBA_SIZE(st_cblk_off), FBA_SIZE(st_cblk_len));
6377 cur_fba_len -= st_cblk_len;
6378 cc_ent1 = cc_ent1->cc_chain;
6379 cc_ent2 = cc_ent2->cc_chain;
6380
6381 while (cur_fba_len > (nsc_size_t)end_cblk_len) {
6382 bcopy(cc_ent1->cc_data, cc_ent2->cc_data, CACHE_BLOCK_SIZE);
6383 cc_ent1 = cc_ent1->cc_chain;
6384 cc_ent2 = cc_ent2->cc_chain;
6385 cur_fba_len -= BLK_FBAS;
6386 }
6387 if (cur_fba_len) {
6388 bcopy(cc_ent1->cc_data, cc_ent2->cc_data,
6389 FBA_SIZE(end_cblk_len));
6390 }
6391
6392 return (NSC_DONE);
6393 }
6394
6395
6396 /*
6397 * _sd_copy_direct - Copies data from one handle direct to another disk.
6398 *
6399 * ARGUMENTS:
6400 * handle1 - handle to read from
6401 * handle2 - handle to write to
6402 * fba_pos1 - disk block number to read from.
6403 * fba_pos2 - disk block number to write to.
6404 * fba_len - length in fbas.
6405 *
6406 * RETURNS:
6407 * errno if return > 0
6408 * NSC_DONE otherwise.
6409 *
6410 * Comments:
6411 * This routine copies data from handle1 directly (sync write)
6412 * onto the disk pointed to by handle2. The handle2 is then
6413 * invalidated since the data it contains is now stale compared to
6414 * the disk.
6415 */
6416
6417 static int
6418 _sd_copy_direct(_sd_buf_handle_t *handle1, _sd_buf_handle_t *handle2,
6419 nsc_off_t fba_pos1, nsc_off_t fba_pos2, nsc_size_t fba_len)
6420 {
6421 int rc;
6422
6423 if (_sdbc_shutdown_in_progress) {
6424 DTRACE_PROBE(shutdown);
6425 return (EIO);
6426 }
6427
6428 if (!_SD_HANDLE_ACTIVE(handle1) || !_SD_HANDLE_ACTIVE(handle2)) {
6429 cmn_err(CE_WARN,
6430 "!sdbc(_sd_copy_direct) handle %p or %p not active",
6431 (void *)handle1, (void *)handle2);
6432
6433 DTRACE_PROBE2(handle_active2, int, handle1->bh_flag,
6434 int, handle2->bh_flag);
6435
6436 return (EINVAL);
6437 }
6438
6439 ASSERT_HANDLE_LIMITS(handle1, fba_pos1, fba_len);
6440 ASSERT_HANDLE_LIMITS(handle2, fba_pos2, fba_len);
6441
6442 if ((handle2->bh_flag & NSC_WRITE) == 0) {
6443 cmn_err(CE_WARN,
6444 "!sdbc(_sd_copy_direct) handle2 %p is not writeable",
6445 (void *)handle2);
6446 DTRACE_PROBE1(handle2_write, int, handle2->bh_flag);
6447 return (EINVAL);
6448 }
6449
6450 rc = _sd_sync_write2(handle2, fba_pos2, fba_len, 0, handle1, fba_pos1);
6451
6452 return (rc);
6453 }
6454
6455
6456 /*
6457 * _sd_enqueue_dirty - Enqueue a list of dirty buffers.
6458 *
6459 * ARGUMENTS:
6460 * cd - cache descriptor.
6461 * chain - pointer to list.
6462 * cc_last - last entry in the chain.
6463 * numq - number of entries in the list.
6464 *
6465 * RETURNS:
6466 * NONE.
6467 *
6468 * Comments:
6469 * This routine queues up the dirty blocks for io processing.
6470 * It uses the cc_last to try to coalesce multiple lists into a
6471 * single list, if consecutive writes are sequential in nature.
6472 */
6473
6474 void
6475 _sd_enqueue_dirty(int cd, _sd_cctl_t *chain, _sd_cctl_t *cc_last, int numq)
6476 {
6477 _sd_cd_info_t *cdi;
6478 _sd_cctl_t *last_ent;
6479 int start_write = 0, maxq = SGIO_MAX;
6480
6481 ASSERT(cd >= 0);
6482 cdi = &(_sd_cache_files[cd]);
6483 #if defined(_SD_DEBUG)
6484 if (chain->cc_dirty_link)
6485 cmn_err(CE_WARN, "!dirty_link set in enq %x fl %x",
6486 chain->cc_dirty_link, chain->cc_flag);
6487 #endif
6488
6489 /* was FAST */
6490 mutex_enter(&(cdi->cd_lock));
6491 cdi->cd_info->sh_numdirty += numq;
6492 if (cc_last == NULL)
6493 numq = 0;
6494
6495 if (cdi->cd_dirty_head == NULL) {
6496 cdi->cd_dirty_head = cdi->cd_dirty_tail = chain;
6497 cdi->cd_last_ent = cc_last;
6498 cdi->cd_lastchain_ptr = chain;
6499 cdi->cd_lastchain = numq;
6500 } else {
6501 if ((cc_last) && (last_ent = cdi->cd_last_ent) &&
6502 (CENTRY_BLK(chain) == (CENTRY_BLK(last_ent)+1)) &&
6503 (SDBC_DIRTY_NEIGHBORS(last_ent, chain)) &&
6504 (cdi->cd_lastchain + numq < maxq)) {
6505 cdi->cd_last_ent->cc_dirty_next = chain;
6506 cdi->cd_last_ent = cc_last;
6507 cdi->cd_lastchain += numq;
6508 } else {
6509 cdi->cd_dirty_tail->cc_dirty_link = chain;
6510 cdi->cd_dirty_tail = chain;
6511 cdi->cd_last_ent = cc_last;
6512 cdi->cd_lastchain_ptr = chain;
6513 cdi->cd_lastchain = numq;
6514 start_write = 1;
6515 }
6516 }
6517 /* was FAST */
6518 mutex_exit(&(cdi->cd_lock));
6519 if (start_write)
6520 (void) _SD_CD_WRITER(cd);
6521 }
6522
6523 /*
6524 * _sd_enqueue_dirty_chain - Enqueue a chain of a list of dirty buffers.
6525 *
6526 * ARGUMENTS:
6527 * cd - cache descriptor.
6528 * chain_first - first list in this chain.
6529 * chain_last - last list in this chain.
6530 * numq - number of entries being queue (total of all lists)
6531 *
6532 * RETURNS:
6533 * NONE.
6534 *
6535 * Comments:
6536 * This routine is called from the processing after io completions.
6537 * If the buffers are still dirty, they are queued up in one shot.
6538 */
6539
6540 void
6541 _sd_enqueue_dirty_chain(int cd,
6542 _sd_cctl_t *chain_first,
6543 _sd_cctl_t *chain_last,
6544 int numq)
6545 {
6546 _sd_cd_info_t *cdi;
6547
6548 ASSERT(cd >= 0);
6549 cdi = &(_sd_cache_files[cd]);
6550 if (chain_last->cc_dirty_link)
6551 cmn_err(CE_PANIC,
6552 "!_sd_enqueue_dirty_chain: chain_last %p dirty_link %p",
6553 (void *)chain_last, (void *)chain_last->cc_dirty_link);
6554 /* was FAST */
6555 mutex_enter(&(cdi->cd_lock));
6556 cdi->cd_last_ent = NULL;
6557 cdi->cd_lastchain_ptr = NULL;
6558 cdi->cd_lastchain = 0;
6559
6560 cdi->cd_info->sh_numdirty += numq;
6561 if (cdi->cd_dirty_head == NULL) {
6562 cdi->cd_dirty_head = chain_first;
6563 cdi->cd_dirty_tail = chain_last;
6564 } else {
6565 cdi->cd_dirty_tail->cc_dirty_link = chain_first;
6566 cdi->cd_dirty_tail = chain_last;
6567 }
6568 /* was FAST */
6569 mutex_exit(&(cdi->cd_lock));
6570 }
6571
6572
6573 #ifndef _MULTI_DATAMODEL
6574 /* ARGSUSED */
6575 #endif
6576 static int
6577 convert_stats(_sd_stats32_t *uptr)
6578 /*
6579 * Convert the 64 bit statistic structure to 32bit version.
6580 * Possibly losing information when cache is > 4gb. Ha!
6581 *
6582 * NOTE: this code isn't really MT ready since the copied to struct
6583 * is static. However the race is pretty benign and isn't a whole
6584 * lot worse than the vanilla version which copies data to user
6585 * space from kernel structures that can be changing under it too.
6586 * We can't use a local stack structure since the data size is
6587 * 70k or so and kernel stacks are tiny (8k).
6588 */
6589 {
6590 #ifndef _MULTI_DATAMODEL
6591 return (SDBC_EMODELCONVERT);
6592 #else
6593 int rc = 0;
6594
6595 /*
6596 * This could be done in less code with bcopy type operations
6597 * but this is simpler to follow and easier to change if
6598 * the structures change.
6599 */
6600
6601 _sd_cache_stats32->net_dirty = _sd_cache_stats->net_dirty;
6602 _sd_cache_stats32->net_pending = _sd_cache_stats->net_pending;
6603 _sd_cache_stats32->net_free = _sd_cache_stats->net_free;
6604 _sd_cache_stats32->st_count = _sd_cache_stats->st_count;
6605 _sd_cache_stats32->st_loc_count = _sd_cache_stats->st_loc_count;
6606 _sd_cache_stats32->st_rdhits = _sd_cache_stats->st_rdhits;
6607 _sd_cache_stats32->st_rdmiss = _sd_cache_stats->st_rdmiss;
6608 _sd_cache_stats32->st_wrhits = _sd_cache_stats->st_wrhits;
6609 _sd_cache_stats32->st_wrmiss = _sd_cache_stats->st_wrmiss;
6610 _sd_cache_stats32->st_blksize = _sd_cache_stats->st_blksize;
6611
6612 _sd_cache_stats32->st_lru_blocks = _sd_cache_stats->st_lru_blocks;
6613 _sd_cache_stats32->st_lru_noreq = _sd_cache_stats->st_lru_noreq;
6614 _sd_cache_stats32->st_lru_req = _sd_cache_stats->st_lru_req;
6615
6616 _sd_cache_stats32->st_wlru_inq = _sd_cache_stats->st_wlru_inq;
6617
6618 _sd_cache_stats32->st_cachesize = _sd_cache_stats->st_cachesize;
6619 _sd_cache_stats32->st_numblocks = _sd_cache_stats->st_numblocks;
6620 _sd_cache_stats32->st_wrcancelns = _sd_cache_stats->st_wrcancelns;
6621 _sd_cache_stats32->st_destaged = _sd_cache_stats->st_destaged;
6622
6623 /*
6624 * bcopy the shared stats which has nothing that needs conversion
6625 * in them
6626 */
6627
6628 bcopy(_sd_cache_stats->st_shared, _sd_cache_stats32->st_shared,
6629 sizeof (_sd_shared_t) * sdbc_max_devs);
6630
6631 if (copyout(_sd_cache_stats32, uptr, sizeof (_sd_stats32_t) +
6632 (sdbc_max_devs - 1) * sizeof (_sd_shared_t)))
6633 rc = EFAULT;
6634
6635 return (rc);
6636 #endif /* _MULTI_DATAMODEL */
6637 }
6638
6639
6640 int
6641 _sd_get_stats(_sd_stats_t *uptr, int convert_32)
6642 {
6643 int rc = 0;
6644
6645 if (_sd_cache_stats == NULL) {
6646 static _sd_stats_t dummy;
6647 #ifdef _MULTI_DATAMODEL
6648 static _sd_stats32_t dummy32;
6649 #endif
6650
6651 if (convert_32) {
6652 #ifdef _MULTI_DATAMODEL
6653 if (copyout(&dummy32, uptr, sizeof (_sd_stats32_t)))
6654 rc = EFAULT;
6655 #else
6656 rc = SDBC_EMODELCONVERT;
6657 #endif
6658 } else if (copyout(&dummy, uptr, sizeof (_sd_stats_t)))
6659 rc = EFAULT;
6660 return (rc);
6661 }
6662
6663 _sd_cache_stats->st_lru_blocks = _sd_lru_q.sq_inq;
6664 _sd_cache_stats->st_lru_noreq = _sd_lru_q.sq_noreq_stat;
6665 _sd_cache_stats->st_lru_req = _sd_lru_q.sq_req_stat;
6666
6667 if (sdbc_safestore) {
6668 ssioc_stats_t ss_stats;
6669
6670 if (SSOP_CTL(sdbc_safestore, SSIOC_STATS,
6671 (uintptr_t)&ss_stats) == 0)
6672 _sd_cache_stats->st_wlru_inq = ss_stats.wq_inq;
6673 else
6674 _sd_cache_stats->st_wlru_inq = 0;
6675 }
6676
6677 if (convert_32)
6678 rc = convert_stats((_sd_stats32_t *)uptr);
6679 else if (copyout(_sd_cache_stats, uptr,
6680 sizeof (_sd_stats_t) + (sdbc_max_devs - 1) * sizeof (_sd_shared_t)))
6681 rc = EFAULT;
6682
6683 return (rc);
6684 }
6685
6686
6687 int
6688 _sd_set_hint(int cd, uint_t hint)
6689 {
6690 int ret = 0;
6691 if (FILE_OPENED(cd)) {
6692 SDTRACE(ST_ENTER|SDF_HINT, cd, 1, SDT_INV_BL, hint, 0);
6693 _sd_cache_files[cd].cd_hint |= (hint & _SD_HINT_MASK);
6694 SDTRACE(ST_EXIT|SDF_HINT, cd, 1, SDT_INV_BL, hint, ret);
6695 } else
6696 ret = EINVAL;
6697
6698 return (ret);
6699 }
6700
6701
6702
6703 int
6704 _sd_clear_hint(int cd, uint_t hint)
6705 {
6706 int ret = 0;
6707 if (FILE_OPENED(cd)) {
6708 SDTRACE(ST_ENTER|SDF_HINT, cd, 2, SDT_INV_BL, hint, 0);
6709 _sd_cache_files[cd].cd_hint &= ~(hint & _SD_HINT_MASK);
6710 SDTRACE(ST_EXIT|SDF_HINT, cd, 2, SDT_INV_BL, hint, ret);
6711 } else
6712 ret = EINVAL;
6713
6714 return (ret);
6715 }
6716
6717
6718 int
6719 _sd_get_cd_hint(int cd, uint_t *hint)
6720 {
6721 *hint = 0;
6722 if (FILE_OPENED(cd)) {
6723 *hint = _sd_cache_files[cd].cd_hint;
6724 return (0);
6725 } else
6726 return (EINVAL);
6727 }
6728
6729 static int
6730 _sd_node_hint_caller(blind_t hint, int hint_action)
6731 {
6732 int rc;
6733
6734 switch (hint_action) {
6735 case NSC_GET_NODE_HINT:
6736 rc = _sd_get_node_hint((uint_t *)hint);
6737 break;
6738 case NSC_SET_NODE_HINT:
6739 rc = _sd_set_node_hint((uint_t)(unsigned long)hint);
6740 break;
6741 case NSC_CLEAR_NODE_HINT:
6742 rc = _sd_clear_node_hint((uint_t)(unsigned long)hint);
6743 break;
6744 default:
6745 rc = EINVAL;
6746 break;
6747 }
6748
6749 return (rc);
6750 }
6751
6752 int
6753 _sd_set_node_hint(uint_t hint)
6754 {
6755 SDTRACE(ST_ENTER|SDF_HINT, SDT_INV_CD, 3, SDT_INV_BL, hint, 0);
6756 if ((_sd_node_hint & NSC_NO_FORCED_WRTHRU) &&
6757 (hint & NSC_FORCED_WRTHRU))
6758 return (EINVAL);
6759 _sd_node_hint |= (hint & _SD_HINT_MASK);
6760 SDTRACE(ST_EXIT|SDF_HINT, SDT_INV_CD, 3, SDT_INV_BL, hint, 0);
6761 return (0);
6762 }
6763
6764
6765 int
6766 _sd_clear_node_hint(uint_t hint)
6767 {
6768 SDTRACE(ST_ENTER|SDF_HINT, SDT_INV_CD, 4, SDT_INV_BL, hint, 0);
6769 _sd_node_hint &= ~(hint & _SD_HINT_MASK);
6770 SDTRACE(ST_EXIT|SDF_HINT, SDT_INV_CD, 4, SDT_INV_BL, hint, 0);
6771 return (0);
6772 }
6773
6774
6775 int
6776 _sd_get_node_hint(uint_t *hint)
6777 {
6778 *hint = _sd_node_hint;
6779 return (0);
6780 }
6781
6782
6783 int
6784 _sd_get_partsize(blind_t xcd, nsc_size_t *ptr)
6785 {
6786 int cd = (int)(unsigned long)xcd;
6787
6788 if (FILE_OPENED(cd)) {
6789 *ptr = _sd_cache_files[cd].cd_info->sh_filesize;
6790 return (0);
6791 } else
6792 return (EINVAL);
6793 }
6794
6795
6796 int
6797 _sd_get_maxfbas(blind_t xcd, int flag, nsc_size_t *ptr)
6798 {
6799 int cd = (int)(unsigned long)xcd;
6800
6801 if (!FILE_OPENED(cd))
6802 return (EINVAL);
6803
6804 if (flag & NSC_CACHEBLK)
6805 *ptr = BLK_FBAS;
6806 else
6807 *ptr = sdbc_max_fbas;
6808
6809 return (0);
6810 }
6811
6812
6813 int
6814 _sd_control(blind_t xcd, int cmd, void *ptr, int len)
6815 {
6816 _sd_cd_info_t *cdi;
6817 int cd = (int)(unsigned long)xcd;
6818
6819 cdi = &(_sd_cache_files[cd]);
6820 return (nsc_control(cdi->cd_rawfd, cmd, ptr, len));
6821 }
6822
6823
6824 int
6825 _sd_discard_pinned(blind_t xcd, nsc_off_t fba_pos, nsc_size_t fba_len)
6826 {
6827 int cd = (int)(unsigned long)xcd;
6828 _sd_cctl_t *cc_ent, **cc_lst, **cc_tmp, *nxt;
6829 ss_centry_info_t *wctl;
6830 int found = 0;
6831 nsc_off_t cblk;
6832 _sd_cd_info_t *cdi = &_sd_cache_files[cd];
6833 int rc;
6834
6835 if ((!FILE_OPENED(cd)) || (!cdi->cd_info->sh_failed)) {
6836
6837 return (EINVAL);
6838 }
6839
6840 for (cblk = FBA_TO_BLK_NUM(fba_pos);
6841 cblk < FBA_TO_BLK_LEN(fba_pos + fba_len); cblk++) {
6842 if (cc_ent =
6843 (_sd_cctl_t *)_sd_hash_search(cd, cblk, _sd_htable)) {
6844 if (!CENTRY_PINNED(cc_ent))
6845 continue;
6846
6847 /*
6848 * remove cc_ent from failed links
6849 * cc_lst - pointer to "cc_dirty_link" pointer
6850 * starts at &cd_failed_head.
6851 * cc_tmp - pointer to "cc_dirty_next"
6852 * except when equal to cc_lst.
6853 */
6854 mutex_enter(&cdi->cd_lock);
6855 cc_tmp = cc_lst = &(cdi->cd_fail_head);
6856 while (*cc_tmp != cc_ent) {
6857 cc_tmp = &((*cc_tmp)->cc_dirty_next);
6858 if (!*cc_tmp)
6859 cc_lst = &((*cc_lst)->cc_dirty_link),
6860 cc_tmp = cc_lst;
6861 }
6862 if (*cc_tmp) {
6863 found++;
6864 if (cc_lst != cc_tmp) /* break chain */
6865 *cc_tmp = NULL;
6866 nxt = cc_ent->cc_dirty_next;
6867 if (nxt) {
6868 nxt->cc_dirty_link =
6869 (*cc_lst)->cc_dirty_link;
6870 *cc_lst = nxt;
6871 } else {
6872 *cc_lst = (*cc_lst)->cc_dirty_link;
6873 }
6874 cdi->cd_info->sh_numfail--;
6875 nsc_unpinned_data(cdi->cd_iodev,
6876 BLK_TO_FBA_NUM(CENTRY_BLK(cc_ent)),
6877 BLK_FBAS);
6878 }
6879 mutex_exit(&cdi->cd_lock);
6880
6881 /* clear dirty bits */
6882 /* was FAST */
6883 mutex_enter(&cc_ent->cc_lock);
6884 cc_ent->cc_valid = cc_ent->cc_dirty = 0;
6885 cc_ent->cc_flag &= ~(CC_QHEAD|CC_PEND_DIRTY|CC_PINNED);
6886 cc_ent->cc_dirty_link = NULL;
6887 wctl = cc_ent->cc_write;
6888 cc_ent->cc_write = NULL;
6889 /* was FAST */
6890 mutex_exit(&cc_ent->cc_lock);
6891
6892 /* release cache block to head of LRU */
6893 if (wctl) {
6894 wctl->sc_flag = 0;
6895 wctl->sc_dirty = 0;
6896 SSOP_SETCENTRY(sdbc_safestore, wctl);
6897 SSOP_DEALLOCRESOURCE(sdbc_safestore,
6898 wctl->sc_res);
6899 }
6900
6901 if (!sdbc_use_dmchain)
6902 _sd_requeue_head(cc_ent);
6903 }
6904 }
6905
6906 rc = found ? NSC_DONE : EINVAL;
6907
6908 return (rc);
6909 }
6910
6911
6912 /*
6913 * Handle allocation
6914 */
6915
6916 _sd_buf_hlist_t _sd_handle_list;
6917
6918 /*
6919 * _sdbc_handles_unload - cache is being unloaded.
6920 */
6921 void
6922 _sdbc_handles_unload(void)
6923 {
6924 mutex_destroy(&_sd_handle_list.hl_lock);
6925
6926 }
6927
6928 /*
6929 * _sdbc_handles_load - cache is being unloaded.
6930 */
6931 int
6932 _sdbc_handles_load(void)
6933 {
6934 mutex_init(&_sd_handle_list.hl_lock, NULL, MUTEX_DRIVER, NULL);
6935
6936 return (0);
6937 }
6938
6939 int
6940 _sdbc_handles_configure()
6941 {
6942 _sd_handle_list.hl_count = 0;
6943
6944 _sd_handle_list.hl_top.bh_next = &_sd_handle_list.hl_top;
6945 _sd_handle_list.hl_top.bh_prev = &_sd_handle_list.hl_top;
6946
6947 return (0);
6948 }
6949
6950
6951
6952 /*
6953 * _sdbc_handles_deconfigure - cache is being deconfigured
6954 */
6955 void
6956 _sdbc_handles_deconfigure(void)
6957 {
6958 _sd_handle_list.hl_count = 0;
6959 }
6960
6961
6962 _sd_buf_handle_t *
6963 _sd_alloc_handle(sdbc_callback_fn_t d_cb, sdbc_callback_fn_t r_cb,
6964 sdbc_callback_fn_t w_cb)
6965 {
6966 _sd_buf_handle_t *handle;
6967
6968 handle = (_sd_buf_handle_t *)kmem_zalloc(sizeof (_sd_buf_handle_t),
6969 KM_SLEEP);
6970 /* maintain list and count for debugging */
6971 mutex_enter(&_sd_handle_list.hl_lock);
6972
6973 handle->bh_prev = &_sd_handle_list.hl_top;
6974 handle->bh_next = _sd_handle_list.hl_top.bh_next;
6975 _sd_handle_list.hl_top.bh_next->bh_prev = handle;
6976 _sd_handle_list.hl_top.bh_next = handle;
6977
6978 ++_sd_handle_list.hl_count;
6979 mutex_exit(&_sd_handle_list.hl_lock);
6980 #if !defined(_SD_NOCHECKS)
6981 ASSERT(!(handle->bh_flag & (NSC_HALLOCATED | NSC_HACTIVE)));
6982 #endif
6983 handle->bh_disconnect_cb = d_cb;
6984 handle->bh_read_cb = r_cb;
6985 handle->bh_write_cb = w_cb;
6986 handle->bh_flag |= NSC_HALLOCATED;
6987 handle->bh_alloc_thread = nsc_threadp();
6988
6989 return (handle);
6990 }
6991
6992 int
6993 _sd_free_handle(_sd_buf_handle_t *handle)
6994 {
6995
6996 if ((handle->bh_flag & NSC_HALLOCATED) == 0) {
6997 cmn_err(CE_WARN, "!sdbc(_sd_free_handle) handle %p not valid",
6998 (void *)handle);
6999
7000 DTRACE_PROBE(_sd_free_handle_end);
7001
7002 return (EINVAL);
7003 }
7004 if (_SD_HANDLE_ACTIVE(handle)) {
7005 cmn_err(CE_WARN,
7006 "!sdbc(_sd_free_handle) attempt to free active handle %p",
7007 (void *)handle);
7008
7009 DTRACE_PROBE1(free_handle_active, int, handle->bh_flag);
7010
7011 return (EINVAL);
7012 }
7013
7014
7015 /* remove from queue before free */
7016 mutex_enter(&_sd_handle_list.hl_lock);
7017 handle->bh_prev->bh_next = handle->bh_next;
7018 handle->bh_next->bh_prev = handle->bh_prev;
7019 --_sd_handle_list.hl_count;
7020 mutex_exit(&_sd_handle_list.hl_lock);
7021
7022 kmem_free(handle, sizeof (_sd_buf_handle_t));
7023
7024 return (0);
7025 }
7026
7027
7028
7029
7030 #if !defined (_SD_8K_BLKSIZE)
7031 #define _SD_MAX_MAP 0x100
7032 #else /* !(_SD_8K_BLKSIZE) */
7033 #define _SD_MAX_MAP 0x10000
7034 #endif /* !(_SD_8K_BLKSIZE) */
7035
7036 char _sd_contig_bmap[_SD_MAX_MAP];
7037 _sd_map_info_t _sd_lookup_map[_SD_MAX_MAP];
7038
7039 void
7040 _sd_init_contig_bmap(void)
7041 {
7042 int i, j;
7043
7044 for (i = 1; i < _SD_MAX_MAP; i = ((i << 1) | 1))
7045 for (j = i; j < _SD_MAX_MAP; j <<= 1)
7046 _sd_contig_bmap[j] = 1;
7047 }
7048
7049
7050
7051
7052 void
7053 _sd_init_lookup_map(void)
7054 {
7055 unsigned int i, j, k;
7056 int stpos, len;
7057 _sd_bitmap_t mask;
7058
7059 for (i = 0; i < _SD_MAX_MAP; i++) {
7060 for (j = i, k = 0; j && ((j & 1) == 0); j >>= 1, k++)
7061 ;
7062 stpos = k;
7063 _sd_lookup_map[i].mi_stpos = (unsigned char)k;
7064
7065 for (k = 0; j & 1; j >>= 1, k++)
7066 ;
7067 len = k;
7068 _sd_lookup_map[i].mi_len = (unsigned char)k;
7069
7070 _sd_lookup_map[i].mi_mask = SDBC_GET_BITS(stpos, len);
7071 }
7072 for (i = 0; i < _SD_MAX_MAP; i++) {
7073 mask = (_sd_bitmap_t)i;
7074 for (j = 0; mask; j++)
7075 SDBC_LOOKUP_MODIFY(mask);
7076
7077 _sd_lookup_map[i].mi_dirty_count = (unsigned char)j;
7078 }
7079 for (i = 0; i < _SD_MAX_MAP; i++) {
7080 _sd_lookup_map[i].mi_io_count = SDBC_LOOKUP_DTCOUNT(i);
7081 mask = ~i;
7082 _sd_lookup_map[i].mi_io_count += SDBC_LOOKUP_DTCOUNT(mask);
7083 }
7084 }
7085
7086
7087 nsc_def_t _sd_sdbc_def[] = {
7088 "Open", (uintptr_t)_sd_open_io, 0,
7089 "Close", (uintptr_t)_sd_close_io, 0,
7090 "Attach", (uintptr_t)_sdbc_io_attach_cd, 0,
7091 "Detach", (uintptr_t)_sdbc_io_detach_cd, 0,
7092 "AllocBuf", (uintptr_t)_sd_alloc_buf, 0,
7093 "FreeBuf", (uintptr_t)_sd_free_buf, 0,
7094 "Read", (uintptr_t)_sd_read, 0,
7095 "Write", (uintptr_t)_sd_write, 0,
7096 "Zero", (uintptr_t)_sd_zero, 0,
7097 "Copy", (uintptr_t)_sd_copy, 0,
7098 "CopyDirect", (uintptr_t)_sd_copy_direct, 0,
7099 "Uncommit", (uintptr_t)_sd_uncommit, 0,
7100 "AllocHandle", (uintptr_t)_sd_alloc_handle, 0,
7101 "FreeHandle", (uintptr_t)_sd_free_handle, 0,
7102 "Discard", (uintptr_t)_sd_discard_pinned, 0,
7103 "Sizes", (uintptr_t)_sd_cache_sizes, 0,
7104 "GetPinned", (uintptr_t)_sd_get_pinned, 0,
7105 "NodeHints", (uintptr_t)_sd_node_hint_caller, 0,
7106 "PartSize", (uintptr_t)_sd_get_partsize, 0,
7107 "MaxFbas", (uintptr_t)_sd_get_maxfbas, 0,
7108 "Control", (uintptr_t)_sd_control, 0,
7109 "Provide", NSC_CACHE, 0,
7110 0, 0, 0
7111 };
7112
7113 /*
7114 * do the SD_GET_CD_CLUSTER_DATA ioctl (get the global filename data)
7115 */
7116 /* ARGSUSED */
7117 int
7118 sd_get_file_info_data(char *uaddrp)
7119 {
7120 return (ENOTTY);
7121 }
7122
7123 /*
7124 * do the SD_GET_CD_CLUSTER_SIZE ioctl (get size of global filename area)
7125 */
7126 int
7127 sd_get_file_info_size(void *uaddrp)
7128 {
7129 if (copyout(&_sdbc_gl_file_info_size, uaddrp,
7130 sizeof (_sdbc_gl_file_info_size))) {
7131 return (EFAULT);
7132 }
7133
7134 return (0);
7135 }
7136
7137
7138 /*
7139 * SD_GET_GLMUL_SIZES ioctl
7140 * get sizes of the global info regions (for this node only)
7141 */
7142 /* ARGSUSED */
7143 int
7144 sd_get_glmul_sizes(int *uaddrp)
7145 {
7146 return (ENOTTY);
7147 }
7148
7149 /*
7150 * SD_GET_GLMUL_INFO ioctl
7151 * get the global metadata for write blocks (for this node only)
7152 */
7153 /* ARGSUSED */
7154 int
7155 sd_get_glmul_info(char *uaddrp)
7156 {
7157
7158 return (ENOTTY);
7159 }
7160
7161 int
7162 sdbc_global_stats_update(kstat_t *ksp, int rw)
7163 {
7164 sdbc_global_stats_t *sdbc_gstats;
7165 _sd_stats_t *gstats_vars;
7166 uint_t hint;
7167
7168 sdbc_gstats = (sdbc_global_stats_t *)(ksp->ks_data);
7169
7170 gstats_vars = _sd_cache_stats;
7171
7172 if (rw == KSTAT_WRITE) {
7173 return (EACCES);
7174 }
7175
7176 /* default to READ */
7177 sdbc_gstats->ci_sdbc_count.value.ul = gstats_vars->st_count;
7178 sdbc_gstats->ci_sdbc_loc_count.value.ul = gstats_vars->st_loc_count;
7179 sdbc_gstats->ci_sdbc_rdhits.value.ul = (ulong_t)gstats_vars->st_rdhits;
7180 sdbc_gstats->ci_sdbc_rdmiss.value.ul = (ulong_t)gstats_vars->st_rdmiss;
7181 sdbc_gstats->ci_sdbc_wrhits.value.ul = (ulong_t)gstats_vars->st_wrhits;
7182 sdbc_gstats->ci_sdbc_wrmiss.value.ul = (ulong_t)gstats_vars->st_wrmiss;
7183
7184 sdbc_gstats->ci_sdbc_blksize.value.ul =
7185 (ulong_t)gstats_vars->st_blksize;
7186 sdbc_gstats->ci_sdbc_lru_blocks.value.ul = (ulong_t)_sd_lru_q.sq_inq;
7187 #ifdef DEBUG
7188 sdbc_gstats->ci_sdbc_lru_noreq.value.ul =
7189 (ulong_t)_sd_lru_q.sq_noreq_stat;
7190 sdbc_gstats->ci_sdbc_lru_req.value.ul = (ulong_t)_sd_lru_q.sq_req_stat;
7191 #endif
7192 sdbc_gstats->ci_sdbc_wlru_inq.value.ul =
7193 (ulong_t)gstats_vars->st_wlru_inq;
7194 sdbc_gstats->ci_sdbc_cachesize.value.ul =
7195 (ulong_t)gstats_vars->st_cachesize;
7196 sdbc_gstats->ci_sdbc_numblocks.value.ul =
7197 (ulong_t)gstats_vars->st_numblocks;
7198 sdbc_gstats->ci_sdbc_wrcancelns.value.ul =
7199 (ulong_t)gstats_vars->st_wrcancelns;
7200 sdbc_gstats->ci_sdbc_destaged.value.ul =
7201 (ulong_t)gstats_vars->st_destaged;
7202 sdbc_gstats->ci_sdbc_num_shared.value.ul = (ulong_t)sdbc_max_devs;
7203 (void) _sd_get_node_hint(&hint);
7204 sdbc_gstats->ci_sdbc_nodehints.value.ul = (ulong_t)hint;
7205
7206
7207 return (0);
7208 }
7209
7210 int
7211 sdbc_cd_stats_update(kstat_t *ksp, int rw)
7212 {
7213 sdbc_cd_stats_t *sdbc_shstats;
7214 _sd_shared_t *shstats_vars;
7215 int name_len;
7216 uint_t hint;
7217
7218 sdbc_shstats = (sdbc_cd_stats_t *)(ksp->ks_data);
7219
7220 shstats_vars = (_sd_shared_t *)(ksp->ks_private);
7221
7222 if (rw == KSTAT_WRITE) {
7223 return (EACCES);
7224 }
7225
7226 /* copy tail of filename to kstat. leave 1 byte for null char */
7227 if (shstats_vars->sh_filename != NULL) {
7228 name_len = (int)strlen(shstats_vars->sh_filename);
7229 name_len -= (KSTAT_DATA_CHAR_LEN - 1);
7230
7231 if (name_len < 0) {
7232 name_len = 0;
7233 }
7234
7235 (void) strlcpy(sdbc_shstats->ci_sdbc_vol_name.value.c,
7236 shstats_vars->sh_filename + name_len, KSTAT_DATA_CHAR_LEN);
7237 } else {
7238 cmn_err(CE_WARN, "!Kstat error: no volume name associated "
7239 "with cache descriptor");
7240 }
7241
7242 sdbc_shstats->ci_sdbc_failed.value.ul =
7243 (ulong_t)shstats_vars->sh_failed;
7244 sdbc_shstats->ci_sdbc_cd.value.ul = (ulong_t)shstats_vars->sh_cd;
7245 sdbc_shstats->ci_sdbc_cache_read.value.ul =
7246 (ulong_t)shstats_vars->sh_cache_read;
7247 sdbc_shstats->ci_sdbc_cache_write.value.ul =
7248 (ulong_t)shstats_vars->sh_cache_write;
7249 sdbc_shstats->ci_sdbc_disk_read.value.ul =
7250 (ulong_t)shstats_vars->sh_disk_read;
7251 sdbc_shstats->ci_sdbc_disk_write.value.ul =
7252 (ulong_t)shstats_vars->sh_disk_write;
7253 #ifdef NSC_MULTI_TERABYTE
7254 sdbc_shstats->ci_sdbc_filesize.value.ui64 =
7255 (uint64_t)shstats_vars->sh_filesize;
7256 #else
7257 sdbc_shstats->ci_sdbc_filesize.value.ul =
7258 (ulong_t)shstats_vars->sh_filesize;
7259 #endif
7260 sdbc_shstats->ci_sdbc_numdirty.value.ul =
7261 (ulong_t)shstats_vars->sh_numdirty;
7262 sdbc_shstats->ci_sdbc_numio.value.ul = (ulong_t)shstats_vars->sh_numio;
7263 sdbc_shstats->ci_sdbc_numfail.value.ul =
7264 (ulong_t)shstats_vars->sh_numfail;
7265 sdbc_shstats->ci_sdbc_destaged.value.ul =
7266 (ulong_t)shstats_vars->sh_destaged;
7267 sdbc_shstats->ci_sdbc_wrcancelns.value.ul =
7268 (ulong_t)shstats_vars->sh_wrcancelns;
7269 (void) _sd_get_cd_hint(shstats_vars->sh_cd, &hint);
7270 sdbc_shstats->ci_sdbc_cdhints.value.ul = (ulong_t)hint;
7271
7272
7273 return (0);
7274 }
7275
7276
7277 /*
7278 * cd_kstat_add
7279 *
7280 * Installs all kstats and associated infrastructure (mutex, buffer),
7281 * associated with a particular cache descriptor. This function is called
7282 * when the cache descriptor is opened in _sd_open().
7283 * "cd" -- cache descriptor number whose kstats we wish to add
7284 * returns: 0 on success, -1 on failure
7285 */
7286 static int
7287 cd_kstat_add(int cd)
7288 {
7289 char name[KSTAT_STRLEN];
7290
7291 if (cd < 0 || cd >= sdbc_max_devs) {
7292 cmn_err(CE_WARN, "!invalid cache descriptor: %d", cd);
7293 return (-1);
7294 }
7295
7296 /* create a regular kstat for this cache descriptor */
7297 if (!sdbc_cd_kstats) {
7298 cmn_err(CE_WARN, "!sdbc_cd_kstats not allocated");
7299 return (-1);
7300 }
7301
7302 (void) snprintf(name, KSTAT_STRLEN, "%s%d", SDBC_KSTAT_CDSTATS, cd);
7303
7304 sdbc_cd_kstats[cd] = kstat_create(SDBC_KSTAT_MODULE,
7305 cd, name, SDBC_KSTAT_CLASS, KSTAT_TYPE_NAMED,
7306 sizeof (sdbc_cd_stats)/sizeof (kstat_named_t),
7307 KSTAT_FLAG_VIRTUAL|KSTAT_FLAG_WRITABLE);
7308
7309 if (sdbc_cd_kstats[cd] != NULL) {
7310 sdbc_cd_kstats[cd]->ks_data = &sdbc_cd_stats;
7311 sdbc_cd_kstats[cd]->ks_update = sdbc_cd_stats_update;
7312 sdbc_cd_kstats[cd]->ks_private =
7313 &_sd_cache_stats->st_shared[cd];
7314 kstat_install(sdbc_cd_kstats[cd]);
7315 } else {
7316 cmn_err(CE_WARN, "!cdstats %d kstat allocation failed", cd);
7317 }
7318
7319 /* create an I/O kstat for this cache descriptor */
7320 if (!sdbc_cd_io_kstats) {
7321 cmn_err(CE_WARN, "!sdbc_cd_io_kstats not allocated");
7322 return (-1);
7323 }
7324
7325 (void) snprintf(name, KSTAT_STRLEN, "%s%d", SDBC_IOKSTAT_CDSTATS, cd);
7326
7327 sdbc_cd_io_kstats[cd] = kstat_create(
7328 SDBC_KSTAT_MODULE, cd, name, "disk", KSTAT_TYPE_IO, 1, 0);
7329
7330 if (sdbc_cd_io_kstats[cd]) {
7331 if (!sdbc_cd_io_kstats_mutexes) {
7332 cmn_err(CE_WARN, "!sdbc_cd_io_kstats_mutexes not "
7333 "allocated");
7334 return (-1);
7335 }
7336
7337 mutex_init(&sdbc_cd_io_kstats_mutexes[cd], NULL,
7338 MUTEX_DRIVER, NULL);
7339
7340 sdbc_cd_io_kstats[cd]->ks_lock = &sdbc_cd_io_kstats_mutexes[cd];
7341
7342 kstat_install(sdbc_cd_io_kstats[cd]);
7343
7344 } else {
7345 cmn_err(CE_WARN, "!sdbc cd %d io kstat allocation failed", cd);
7346 }
7347
7348 return (0);
7349 }
7350
7351 /*
7352 * cd_kstat_remove
7353 *
7354 * Uninstalls all kstats and associated infrastructure (mutex, buffer),
7355 * associated with a particular cache descriptor. This function is called
7356 * when the cache descriptor is closed in _sd_close().
7357 * "cd" -- cache descriptor number whose kstats we wish to remove
7358 * returns: 0 on success, -1 on failure
7359 */
7360 static int
7361 cd_kstat_remove(int cd)
7362 {
7363 if (cd < 0 || cd >= sdbc_max_devs) {
7364 cmn_err(CE_WARN, "!invalid cache descriptor: %d", cd);
7365 return (-1);
7366 }
7367
7368 /* delete the regular kstat corresponding to this cache descriptor */
7369 if (sdbc_cd_kstats && sdbc_cd_kstats[cd]) {
7370 kstat_delete(sdbc_cd_kstats[cd]);
7371 sdbc_cd_kstats[cd] = NULL;
7372 }
7373
7374 /* delete the I/O kstat corresponding to this cache descriptor */
7375 if (sdbc_cd_io_kstats && sdbc_cd_io_kstats[cd]) {
7376 kstat_delete(sdbc_cd_io_kstats[cd]);
7377 sdbc_cd_io_kstats[cd] = NULL;
7378
7379 if (sdbc_cd_io_kstats_mutexes) {
7380 /* destroy the mutex associated with this I/O kstat */
7381 mutex_destroy(&sdbc_cd_io_kstats_mutexes[cd]);
7382 }
7383 }
7384
7385 return (0);
7386 }
7387
7388 #ifdef DEBUG
7389 /*
7390 * kstat update
7391 */
7392 int
7393 sdbc_dynmem_kstat_update_dm(kstat_t *ksp, int rw)
7394 {
7395 sdbc_dynmem_dm_t *sdbc_dynmem;
7396 _dm_process_vars_t *process_vars;
7397 _dm_process_vars_t local_dm_process_vars;
7398
7399 simplect_dm++;
7400
7401 sdbc_dynmem = (sdbc_dynmem_dm_t *)(ksp->ks_data);
7402
7403 /* global dynmem_processing_dm */
7404 process_vars = (_dm_process_vars_t *)(ksp->ks_private);
7405
7406 if (rw == KSTAT_WRITE) {
7407 simplect_dm = sdbc_dynmem->ci_sdbc_simplect.value.ul;
7408 local_dm_process_vars.monitor_dynmem_process =
7409 sdbc_dynmem->ci_sdbc_monitor_dynmem.value.ul;
7410 local_dm_process_vars.max_dyn_list =
7411 sdbc_dynmem->ci_sdbc_max_dyn_list.value.ul;
7412 local_dm_process_vars.cache_aging_ct1 =
7413 sdbc_dynmem->ci_sdbc_cache_aging_ct1.value.ul;
7414 local_dm_process_vars.cache_aging_ct2 =
7415 sdbc_dynmem->ci_sdbc_cache_aging_ct2.value.ul;
7416 local_dm_process_vars.cache_aging_ct3 =
7417 sdbc_dynmem->ci_sdbc_cache_aging_ct3.value.ul;
7418 local_dm_process_vars.cache_aging_sec1 =
7419 sdbc_dynmem->ci_sdbc_cache_aging_sec1.value.ul;
7420 local_dm_process_vars.cache_aging_sec2 =
7421 sdbc_dynmem->ci_sdbc_cache_aging_sec2.value.ul;
7422 local_dm_process_vars.cache_aging_sec3 =
7423 sdbc_dynmem->ci_sdbc_cache_aging_sec3.value.ul;
7424 local_dm_process_vars.cache_aging_pcnt1 =
7425 sdbc_dynmem->ci_sdbc_cache_aging_pcnt1.value.ul;
7426 local_dm_process_vars.cache_aging_pcnt2 =
7427 sdbc_dynmem->ci_sdbc_cache_aging_pcnt2.value.ul;
7428 local_dm_process_vars.max_holds_pcnt =
7429 sdbc_dynmem->ci_sdbc_max_holds_pcnt.value.ul;
7430 local_dm_process_vars.process_directive =
7431 sdbc_dynmem->ci_sdbc_process_directive.value.ul;
7432 (void) sdbc_edit_xfer_process_vars_dm(&local_dm_process_vars);
7433
7434 if (process_vars->process_directive & WAKE_DEALLOC_THREAD_DM) {
7435 process_vars->process_directive &=
7436 ~WAKE_DEALLOC_THREAD_DM;
7437 mutex_enter(&dynmem_processing_dm.thread_dm_lock);
7438 cv_broadcast(&dynmem_processing_dm.thread_dm_cv);
7439 mutex_exit(&dynmem_processing_dm.thread_dm_lock);
7440 }
7441
7442 return (0);
7443 }
7444
7445 /* default to READ */
7446 sdbc_dynmem->ci_sdbc_simplect.value.ul = simplect_dm;
7447 sdbc_dynmem->ci_sdbc_monitor_dynmem.value.ul =
7448 process_vars->monitor_dynmem_process;
7449 sdbc_dynmem->ci_sdbc_max_dyn_list.value.ul =
7450 process_vars->max_dyn_list;
7451 sdbc_dynmem->ci_sdbc_cache_aging_ct1.value.ul =
7452 process_vars->cache_aging_ct1;
7453 sdbc_dynmem->ci_sdbc_cache_aging_ct2.value.ul =
7454 process_vars->cache_aging_ct2;
7455 sdbc_dynmem->ci_sdbc_cache_aging_ct3.value.ul =
7456 process_vars->cache_aging_ct3;
7457 sdbc_dynmem->ci_sdbc_cache_aging_sec1.value.ul =
7458 process_vars->cache_aging_sec1;
7459 sdbc_dynmem->ci_sdbc_cache_aging_sec2.value.ul =
7460 process_vars->cache_aging_sec2;
7461 sdbc_dynmem->ci_sdbc_cache_aging_sec3.value.ul =
7462 process_vars->cache_aging_sec3;
7463 sdbc_dynmem->ci_sdbc_cache_aging_pcnt1.value.ul =
7464 process_vars->cache_aging_pcnt1;
7465 sdbc_dynmem->ci_sdbc_cache_aging_pcnt2.value.ul =
7466 process_vars->cache_aging_pcnt2;
7467 sdbc_dynmem->ci_sdbc_max_holds_pcnt.value.ul =
7468 process_vars->max_holds_pcnt;
7469 sdbc_dynmem->ci_sdbc_process_directive.value.ul =
7470 process_vars->process_directive;
7471
7472 sdbc_dynmem->ci_sdbc_alloc_ct.value.ul = process_vars->alloc_ct;
7473 sdbc_dynmem->ci_sdbc_dealloc_ct.value.ul = process_vars->dealloc_ct;
7474 sdbc_dynmem->ci_sdbc_history.value.ul = process_vars->history;
7475 sdbc_dynmem->ci_sdbc_nodatas.value.ul = process_vars->nodatas;
7476 sdbc_dynmem->ci_sdbc_candidates.value.ul = process_vars->candidates;
7477 sdbc_dynmem->ci_sdbc_deallocs.value.ul = process_vars->deallocs;
7478 sdbc_dynmem->ci_sdbc_hosts.value.ul = process_vars->hosts;
7479 sdbc_dynmem->ci_sdbc_pests.value.ul = process_vars->pests;
7480 sdbc_dynmem->ci_sdbc_metas.value.ul = process_vars->metas;
7481 sdbc_dynmem->ci_sdbc_holds.value.ul = process_vars->holds;
7482 sdbc_dynmem->ci_sdbc_others.value.ul = process_vars->others;
7483 sdbc_dynmem->ci_sdbc_notavail.value.ul = process_vars->notavail;
7484
7485 return (0);
7486 }
7487 #endif