1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #ifndef _RDC_IO_H
27 #define _RDC_IO_H
28
29 #ifdef __cplusplus
30 extern "C" {
31 #endif
32
33
34 #include <sys/unistat/spcs_s.h>
35 #ifdef DS_DDICT
36 #define bool_t int
37 #endif
38 #include <sys/nsctl/rdc_prot.h>
39 #include <sys/nsctl/nsctl.h>
40 #include <sys/nsctl/rdc_ioctl.h>
41
42 /*
43 * Definitions for kstats
44 */
45 #define RDC_MKSTAT_MAXSETS "maxsets"
46 #define RDC_MKSTAT_MAXFBAS "maxfbas"
47 #define RDC_MKSTAT_RPC_TIMEOUT "rpc_timeout"
48 #define RDC_MKSTAT_HEALTH_THRES "health_thres"
49 #define RDC_MKSTAT_BITMAP_WRITES "bitmap_writes"
50 #define RDC_MKSTAT_CLNT_COTS_CALLS "clnt_cots_calls"
51 #define RDC_MKSTAT_CLNT_CLTS_CALLS "clnt_clts_calls"
52 #define RDC_MKSTAT_SVC_COTS_CALLS "svc_cots_calls"
53 #define RDC_MKSTAT_SVC_CLTS_CALLS "svc_clts_calls"
54 #define RDC_MKSTAT_BITMAP_REF_DELAY "bitmap_ref_delay"
55
56 #define RDC_IKSTAT_FLAGS "flags"
57 #define RDC_IKSTAT_SYNCFLAGS "syncflags"
58 #define RDC_IKSTAT_BMPFLAGS "bmpflags"
59 #define RDC_IKSTAT_SYNCPOS "syncpos"
60 #define RDC_IKSTAT_VOLSIZE "volsize"
61 #define RDC_IKSTAT_BITSSET "bitsset"
62 #define RDC_IKSTAT_AUTOSYNC "autosync"
63 #define RDC_IKSTAT_MAXQFBAS "maxqfbas"
64 #define RDC_IKSTAT_MAXQITEMS "maxqitems"
65 #define RDC_IKSTAT_FILE "primary_vol"
66 #define RDC_IKSTAT_SECFILE "secondary_vol"
67 #define RDC_IKSTAT_BITMAP "bitmap"
68 #define RDC_IKSTAT_PRIMARY_HOST "primary_host"
69 #define RDC_IKSTAT_SECONDARY_HOST "secondary_host"
70 #define RDC_IKSTAT_TYPE_FLAG "type_flag"
71 #define RDC_IKSTAT_BMP_SIZE "bmp_size"
72 #define RDC_IKSTAT_DISK_STATUS "disk_status"
73 #define RDC_IKSTAT_IF_DOWN "if_down"
74 #define RDC_IKSTAT_IF_RPC_VERSION "if_rpc_version"
75 #define RDC_IKSTAT_ASYNC_THROTTLE_DELAY "async_throttle_delay"
76 #define RDC_IKSTAT_ASYNC_BLOCK_HWM "async_block_hwm"
77 #define RDC_IKSTAT_ASYNC_ITEM_HWM "async_item_hwm"
78 #define RDC_IKSTAT_QUEUE_TYPE "async_queue_type"
79 #define RDC_IKSTAT_ASYNC_ITEMS "async_queue_items"
80 #define RDC_IKSTAT_ASYNC_BLOCKS "async_queue_blocks"
81
82 /*
83 * Queue types
84 */
85 #define RDC_DISKQUE 0X01
86 #define RDC_MEMQUE 0x02
87 #define RDC_NOQUE -1
88
89 #define RDC_ACTIVE 0x1
90 #define RDC_INACTIVE 0x2
91
92 #ifdef _KERNEL
93
94 extern nstset_t *_rdc_ioset;
95 extern nstset_t *_rdc_flset;
96
97 #ifdef DEBUG
98 extern int RDC_MAX_SYNC_THREADS;
99 extern int rdc_maxthreads_last;
100 int num_sync_threads;
101 #else
102 #define RDC_MAX_SYNC_THREADS 8
103 #endif
104 #ifdef DEBUG
105 #define RDC_AVAIL_THR_TUNE(n) \
106 do { \
107 if (rdc_maxthreads_last < RDC_MAX_SYNC_THREADS) { \
108 (void) nst_add_thread(n.rdc_syncset, \
109 RDC_MAX_SYNC_THREADS - rdc_maxthreads_last);\
110 } \
111 if (rdc_maxthreads_last > RDC_MAX_SYNC_THREADS) { \
112 (void) nst_del_thread(n.rdc_syncset, \
113 rdc_maxthreads_last - RDC_MAX_SYNC_THREADS); \
114 } \
115 n.avail_thr = RDC_MAX_SYNC_THREADS - n.active_thr; \
116 if (n.avail_thr < 0) { \
117 n.avail_thr = 0; \
118 } \
119 rdc_maxthreads_last = RDC_MAX_SYNC_THREADS; \
120 num_sync_threads = nst_nthread(n.rdc_syncset); \
121 } while (0);
122 #else
123 #define RDC_AVAIL_THR_TUNE(n) \
124 do { \
125 n.avail_thr = RDC_MAX_SYNC_THREADS - n.active_thr; \
126 if (n.avail_thr < 0) \
127 n.avail_thr = 0; \
128 } while (0);
129
130 #endif
131
132 typedef struct syncloop_info {
133 int active_thr;
134 int avail_thr; /* should be MAX_RDC_SYNC_THREADS - active */
135 kmutex_t lock;
136 nstset_t *rdc_syncset;
137 } sync_info_t;
138
139 sync_info_t sync_info;
140
141 /*
142 * Static server information
143 */
144 typedef struct servinfo {
145 struct knetconfig *ri_knconf; /* bound TLI fd */
146 struct netbuf ri_addr; /* server's address */
147 struct sec_data *ri_secdata; /* sec data for rpcsec module */
148 char *ri_hostname; /* server's hostname */
149 int ri_hostnamelen; /* server's hostname length */
150 } rdc_srv_t;
151
152 /*
153 * Interface structure, including health monitoring.
154 */
155 typedef struct rdc_if_s {
156 struct rdc_if_s *next; /* chain pointer */
157 struct netbuf ifaddr;
158 struct netbuf r_ifaddr;
159 rdc_srv_t *srv; /* servinfo of server end */
160 int if_down; /* i/f is down (set on primary) */
161 int isprimary; /* this end is a primary */
162 int issecondary; /* this end is a secondary */
163 rpcvers_t rpc_version; /* RPC protocol version in use */
164 int no_ping; /* set on secondary to hold off RPCs */
165 int old_pulse; /* previous (current) pulse value */
166 int new_pulse; /* new (incoming) pulse value */
167 int deadness; /* how close to death are we? */
168 volatile int exiting; /* daemon exit flag */
169 time_t last; /* time of last ping */
170 } rdc_if_t;
171
172
173 typedef struct rdc_aio_s {
174 struct rdc_aio_s *next;
175 nsc_buf_t *handle;
176 nsc_buf_t *qhandle;
177 nsc_off_t pos;
178 nsc_off_t qpos;
179 nsc_size_t len;
180 nsc_size_t orig_len;
181 int flag;
182 int iostatus;
183 int index;
184 uint_t seq; /* sequence on async Q */
185 } rdc_aio_t;
186
187 /* values for (rdc_aio_t *)->iostatus */
188 enum {
189 RDC_IO_NONE = 0, /* not used */
190 RDC_IO_INIT, /* io started */
191 RDC_IO_DONE, /* io done successfully */
192 RDC_IO_FAILED, /* io failed */
193 RDC_IO_DISCARDED, /* io discarded */
194 RDC_IO_CANCELLED /* group_log in progress */
195 };
196
197
198 #define RDC_MAX_QBLOCKS 16384 /* 8MB temporary q for diskq to flush to */
199 #define RDC_LOW_QBLOCKS 13927 /* roughly 85% of queue full */
200 #define RDC_HALF_MQUEUE 8192 /* half of the memory queue */
201
202 typedef struct netqueue {
203 rdc_aio_t *net_qhead;
204 rdc_aio_t *net_qtail;
205 kmutex_t net_qlock;
206 int hwmhit; /* queue full hit? reset after hwm */
207 int qfill_sleeping; /* waiting for work? */
208 int qfflags; /* diskq/memq flusher flags */
209 kcondvar_t qfcv; /* for timed waits */
210 volatile nsc_size_t blocks; /* number of FBAs in q */
211 volatile uint64_t nitems; /* number of items in q */
212 volatile int inflbls; /* number of inflight blocks */
213 volatile int inflitems; /* number of inflight items */
214 uint64_t nitems_hwm; /* highest items on queue */
215 nsc_size_t blocks_hwm; /* highest blocks on queue */
216 long throttle_delay; /* Number of times we delayed x 2 */
217 } net_queue;
218
219
220 /*
221 * Bitmap header structures.
222 * These must be fixed size in all data models.
223 * If we ever support little-endian machines (eg. Intel) we will need
224 * to add byte-swapping logic.
225 */
226
227 typedef struct {
228 int32_t magic;
229 int32_t serial_mode;
230 int32_t use_mirror;
231 int32_t mirror_down;
232 int32_t sync_needed;
233 char bitmapname[NSC_MAXPATH];
234 char filename[NSC_MAXPATH];
235 int32_t volume_failed;
236 } rdc_headerv2_t;
237 #define RDC_HDR_V2 0x52444302 /* RDC2 */
238
239 #define RDC_SYNC 0x1
240 #define RDC_REV_SYNC 0x2
241 #define RDC_FULL_SYNC 0x3
242
243 #define RDC_FAILED 0x1
244 #define RDC_COMPLETED 0x2
245
246 typedef struct {
247 char file[NSC_MAXPATH];
248 char bitmap[NSC_MAXPATH];
249 } rdc_hdr_addr_t;
250
251 typedef struct {
252 int32_t magic;
253 rdc_hdr_addr_t primary;
254 rdc_hdr_addr_t secondary;
255 int32_t flags;
256 int32_t autosync;
257 int32_t maxqfbas;
258 int32_t maxqitems;
259 int32_t syshostid; /* for cluster bitmaps */
260 } rdc_headerv3_t;
261 #define RDC_HDR_V3 0x52444303 /* RDC3 */
262
263 typedef struct {
264 int32_t magic;
265 rdc_hdr_addr_t primary;
266 rdc_hdr_addr_t secondary;
267 int32_t flags;
268 int32_t autosync;
269 int32_t maxqfbas;
270 int32_t maxqitems;
271 int32_t syshostid; /* for cluster bitmaps */
272 int32_t asyncthr;
273 } rdc_headerv4_t;
274 #define RDC_HDR_V4 0x52444304 /* RDC4 */
275
276 typedef struct {
277 int32_t magic;
278 rdc_hdr_addr_t primary;
279 rdc_hdr_addr_t secondary;
280 int32_t flags;
281 int32_t autosync;
282 int64_t maxqfbas;
283 int64_t maxqitems;
284 int32_t syshostid; /* for cluster bitmaps */
285 int32_t asyncthr;
286 int32_t refcntsize; /* size in bytes of each refcount */
287 } rdc_headerv5_t;
288 #define RDC_HDR_V5 0x52444305 /* RDC5 */
289
290 typedef rdc_headerv5_t rdc_header_t; /* Current header type */
291 #define RDC_HDR_MAGIC RDC_HDR_V5 /* Current header magic number */
292
293 #endif /* _KERNEL */
294
295 #define RDC_BITMAP_FBA 1 /* Offset at which the bitmap starts */
296 #define RDC_BITREF_FBA(krdc) (RDC_BITMAP_FBA + FBA_LEN(krdc->bitmap_size))
297
298 #ifdef _KERNEL
299
300 #define RDC_FUTILE_ATTEMPTS 50
301 typedef struct aio_buf_s {
302 struct aio_buf_s *next; /* next aio_buf */
303 nsc_buf_t *rdc_abufp; /* actual anon buf */
304 int kindex; /* index we are attached to */
305 } aio_buf_t;
306
307 typedef struct rdc_thrsync {
308 kmutex_t lock;
309 int threads;
310 int complete;
311 kcondvar_t cv;
312 } rdc_thrsync_t;
313
314 typedef struct sync_status_s {
315 int offset;
316 struct sync_status_s *next;
317 } sync_status_t;
318
319 typedef struct rdc_syncthr {
320 nsc_off_t offset;
321 nsc_size_t len;
322 struct rdc_k_info *krdc;
323 sync_status_t *status;
324 } rdc_syncthr_t;
325
326 /*
327 * RDC buffer header
328 */
329
330 typedef struct rdc_buf_s {
331 nsc_buf_t rdc_bufh; /* exported buffer header */
332 nsc_buf_t *rdc_bufp; /* underlying buffer */
333 aio_buf_t *rdc_anon; /* ANON async buffer */
334 struct rdc_fd_s *rdc_fd; /* back link */
335 size_t rdc_vsize; /* size of allocated nsc_vec_t */
336 int rdc_flags; /* flags */
337 kmutex_t aio_lock; /* lock for rdc_anon */
338 rdc_thrsync_t rdc_sync; /* for thread syncronization */
339 } rdc_buf_t;
340
341 #define RDC_VEC_ALLOC 0x1 /* local kmem vector for remote io */
342 #define RDC_ALLOC 0x2 /* rdc_bufp is nsc_buf_alloc'd */
343 #define RDC_ASYNC_VEC 0x4 /* Keep tmp handle for async flusher */
344 #define RDC_REMOTE_BUF 0x8 /* buffer alloc'd for remote io only */
345 #define RDC_NULL_BUF 0x10 /* tell diskq to only store io_hdr */
346 #define RDC_ASYNC_BUF 0x20 /* this buf is to an async vol */
347 #define RDC_NULLBUFREAD 0x0f000000 /* read because RDC_NULL_BUF detected */
348
349 #define BUF_IS_ASYNC(h) (((h) != NULL) && (h)->rdc_flags & RDC_ASYNC_BUF)
350 #define RDC_REMOTE(h) (((h) != NULL) && ((h)->rdc_flags & RDC_REMOTE_BUF) && \
351 (((h)->rdc_flags & RDC_ASYNC_VEC) == 0))
352
353 /* check a handle against a supplied pos/len pair */
354
355 #define RDC_HANDLE_LIMITS(h, p, l) \
356 (((h)->sb_user & RDC_DISKQUE) || \
357 ((p) >= (h)->sb_pos) && \
358 (((p) + (l)) <= ((h)->sb_pos + (h)->sb_len)))
359
360 /* check a dset against a supplied pos/len pair */
361
362 #define RDC_DSET_LIMITS(d, p, l) \
363 (((p) >= (d)->pos) && \
364 (((p) + (l)) <= ((d)->pos + (d)->fbalen)))
365
366 /*
367 * RDC device info structures
368 */
369
370 typedef struct _rdc_info_dev_s {
371 nsc_fd_t *bi_fd; /* file descriptor */
372 nsc_iodev_t *bi_iodev; /* I/O device structure */
373 struct rdc_k_info *bi_krdc; /* back link */
374 int bi_rsrv; /* Count of reserves held */
375 int bi_orsrv; /* Reserves for other io provider */
376 int bi_failed; /* Count of failed (faked) reserves */
377 int bi_ofailed; /* Other io provider failed reserves */
378 int bi_flag; /* Reserve flags */
379 } _rdc_info_dev_t;
380
381
382 typedef struct rdc_info_dev_s {
383 struct rdc_info_dev_s *id_next; /* forward link */
384 _rdc_info_dev_t id_cache_dev; /* cached device info */
385 _rdc_info_dev_t id_raw_dev; /* raw device info */
386 kmutex_t id_rlock; /* reserve/release lock */
387 kcondvar_t id_rcv; /* nsc_release pending cv */
388 int id_sets; /* # of sets referencing */
389 int id_release; /* # of pending nsc_releases */
390 int id_flag; /* flags */
391 } rdc_info_dev_t;
392
393
394 typedef struct rdc_path_s {
395 nsc_path_t *rp_tok; /* nsc_register_path token */
396 int rp_ref; /* # of rdc_fd_t's */
397 } rdc_path_t;
398
399
400 /*
401 * Values for id_flag
402 */
403 #define RDC_ID_CLOSING 0x1 /* device is closing */
404
405 #include <sys/nsctl/rdc_diskq.h>
406
407 /*
408 * value for diskio.seq.
409 */
410 #define RDC_NOSEQ (0) /* ignore sequence */
411 #define RDC_NEWSEQ (1) /* start of sequence */
412
413 typedef struct rdc_sleepq {
414 struct rdc_sleepq *next;
415 uint_t seq; /* sequence in queue */
416 int idx; /* idx number of request */
417 int pindex; /* primary host set index */
418 int sindex; /* secondary host set index */
419 uint64_t qpos; /* offset on primary's queue */
420 int nocache; /* cache flag to alloc_buf */
421 } rdc_sleepq_t;
422
423 /*
424 * RDC group structure
425 */
426 typedef struct rdc_group {
427 int count;
428 int rdc_writer;
429 int unregistering;
430 kmutex_t lock;
431 net_queue ra_queue; /* io todo async queues */
432 kcondvar_t iowaitcv; /* wait for flusher */
433 kcondvar_t unregistercv; /* wait for unregister */
434 int rdc_thrnum; /* number of threads */
435 int rdc_addthrnum; /* number threads added to thr set */
436 kmutex_t addthrnumlk; /* lock for above */
437 rdc_sleepq_t *sleepq; /* head of waiting tasks */
438 /*
439 * Dual use, the outgoing sequence number on the client.
440 * The next expected sequence number on the server.
441 * Protected by the ra_queue lock.
442 */
443 uint_t seq;
444 /*
445 * Dual use, the last acknowledged sequence number.
446 * Used to ensure that the queue doesn't overflow on server
447 * and to stall transmissions on the client.
448 * Protected by the ra_queue lock.
449 */
450 uint_t seqack;
451 int asyncstall; /* count of asleep threads */
452 int asyncdis; /* discard stalled output */
453 kcondvar_t asyncqcv; /* output stall here */
454 int flags; /* memory or disk. status etc */
455 disk_queue diskq; /* disk queue */
456 nsc_fd_t *diskqfd; /* diskq handle */
457 nsc_path_t *q_tok; /* q registration */
458 int diskqrsrv; /* reserve count */
459 kmutex_t diskqmutex; /* enables/disables/reserves */
460 uint_t synccount; /* number of group members syncing */
461 } rdc_group_t;
462
463 /* group state */
464 #define RDC_DISKQ_KILL 0x01 /* a force kill of diskq pending */
465
466 #define RDC_IS_DISKQ(grp) (grp->flags & RDC_DISKQUE)
467 #define RDC_IS_MEMQ(grp) (grp->flags & RDC_MEMQUE)
468
469 /*
470 * These flags are used in the
471 * aux_state field, and are used to track:
472 * AUXSYNCIP: When the code has a sync thread running, used instead
473 * of the RC_SYNCING flag which gets cleared before the sync thread
474 * terminates.
475 * AUXWRITE: Set when rdc_sync_write_thr is running, so the rdc_unintercept
476 * code can wait until a one-to-many write has actually terminated.
477 */
478 #define RDC_AUXSYNCIP 0x01 /* a sync is in progress */
479 #define RDC_AUXWRITE 0x02 /* I've got a write in progress */
480
481
482 /*
483 * RDC kernel-private information
484 */
485 typedef struct rdc_k_info {
486 int index; /* Index into array */
487 int remote_index; /* -1 means unknown */
488 int type_flag;
489 int rpc_version; /* RPC version this set supps */
490 int spare1;
491 nsc_off_t syncbitpos;
492 kmutex_t syncbitmutex; /* lock for syncbitpos */
493 volatile int busy_count; /* ioctls in progress */
494 volatile int sync_done;
495 int aux_state; /* syncing ,don't disable */
496 rdc_thrsync_t syncs; /* _rdc_sync thread tracking */
497 rdc_info_dev_t *devices;
498 nsc_iodev_t *iodev; /* I/O device structure */
499 rdc_path_t cache_path;
500 rdc_path_t raw_path;
501 rdc_if_t *intf;
502 rdc_srv_t *lsrv; /* list of servinfo */
503 nsc_size_t maxfbas; /* returned from nsc_maxfbas */
504 unsigned char *dcio_bitmap;
505 void *bitmap_ref; /* Incore bitmap bit ref */
506 struct rdc_group *group;
507 nsc_size_t bitmap_size;
508 int bmaprsrv; /* bitmap reserve count */
509 int bitmap_write;
510 nsc_fd_t *bitmapfd;
511 nsc_fd_t *remote_fd; /* FCAL direct io */
512 volatile int disk_status; /* set to halt sync */
513 int closing;
514 nsc_path_t *b_tok; /* Bitmap registration */
515 int b_ref;
516 kmutex_t dc_sleep;
517 kmutex_t bmapmutex; /* mutex for bitmap ops */
518 kcondvar_t busycv; /* wait for ioctl to complete */
519 kcondvar_t closingcv; /* unregister_path/close */
520 kcondvar_t haltcv; /* wait for sync to halt */
521 kcondvar_t synccv; /* wait for sync to halt */
522 struct rdc_net_dataset *net_dataset; /* replaces hnds */
523 int64_t io_time; /* moved from cd_info */
524 struct rdc_k_info *many_next; /* 1-to-many circular list */
525 struct rdc_k_info *multi_next; /* to multihop krdc */
526 struct rdc_k_info *group_next; /* group circular list */
527 kstat_t *io_kstats; /* io kstat */
528 kstat_t *bmp_kstats; /* bitmap io kstat */
529 kstat_t *set_kstats; /* set kstat */
530 kmutex_t kstat_mutex; /* mutex for kstats */
531 kmutex_t bmp_kstat_mutex; /* mutex for kstats */
532 struct bm_ref_ops *bm_refs;
533 } rdc_k_info_t;
534
535 #define c_fd devices->id_cache_dev.bi_fd
536 #define c_rsrv devices->id_cache_dev.bi_rsrv
537 #define c_failed devices->id_cache_dev.bi_failed
538 #define c_flag devices->id_cache_dev.bi_flag
539
540 #define c_tok cache_path.rp_tok
541 #define c_ref cache_path.rp_ref
542
543 #define r_fd devices->id_raw_dev.bi_fd
544 #define r_rsrv devices->id_raw_dev.bi_rsrv
545 #define r_failed devices->id_raw_dev.bi_failed
546 #define r_flag devices->id_raw_dev.bi_flag
547
548 #define r_tok raw_path.rp_tok
549 #define r_ref raw_path.rp_ref
550
551 /*
552 * flags for _rdc_rsrv_devs()
553 */
554
555 /*
556 * which device(s) to reserve - integer bitmap.
557 */
558
559 #define RDC_CACHE 0x1 /* data device in cache mode */
560 #define RDC_RAW 0x2 /* data device in raw mode */
561 #define RDC_BMP 0x4 /* bitmap device */
562 #define RDC_QUE 0x8 /* diskq device */
563
564 /*
565 * device usage after reserve - integer flag.
566 */
567
568 #define RDC_INTERNAL 0x1 /* reserve for rdc internal purposes */
569 #define RDC_EXTERNAL 0x2 /* reserve in response to io provider Attach */
570
571 /*
572 * Utility macro for nsc_*() io function returns.
573 */
574
575 #define RDC_SUCCESS(rc) (((rc) == NSC_DONE) || ((rc) == NSC_HIT))
576
577 /*
578 * RDC file descriptor structure
579 */
580
581 typedef struct rdc_fd_s {
582 rdc_k_info_t *rdc_info; /* devices info structure */
583 int rdc_type; /* open type, diskq or bitmap */
584 int rdc_oflags; /* raw or cached open type */
585 } rdc_fd_t;
586
587 /*
588 * fd and rsrv macros
589 */
590
591 #define RSRV(bi) (((bi)->bi_rsrv > 0) || ((bi)->bi_failed > 0))
592 #define ORSRV(bi) (((bi)->bi_orsrv > 0) || ((bi)->bi_ofailed > 0))
593 #define RFAILED(bi) (((bi)->bi_failed > 0) || ((bi)->bi_ofailed > 0))
594
595 #define IS_RSRV(bi) (RSRV(bi) || ORSRV(bi))
596
597 #define IS_CRSRV(gcd) (IS_RSRV(&(gcd)->devices->id_cache_dev))
598 #define IS_RRSRV(gcd) (IS_RSRV(&(gcd)->devices->id_raw_dev))
599
600 #define IS_RFAILED(gcd) \
601 (RFAILED(&(gcd)->devices->id_cache_dev) || \
602 RFAILED(&(gcd)->devices->id_raw_dev))
603
604 #define RDC_IS_BMP(rdc) ((rdc)->rdc_type == RDC_BMP)
605 #define RDC_IS_QUE(rdc) ((rdc)->rdc_type == RDC_QUE)
606 #define RDC_IS_RAW(rdc) (((rdc)->rdc_oflags & NSC_CACHE) == 0)
607 #define RDC_U_FD(gcd) (IS_CRSRV(gcd) ? (gcd)->c_fd : (gcd)->r_fd)
608 #define RDC_FD(rdc) (RDC_U_FD(rdc->rdc_info))
609
610
611 typedef struct rdc_host_u {
612 char *nodename;
613 int netaddr;
614 struct netbuf *naddr;
615 } rdc_host_t;
616
617 /*
618 * Reply from remote read
619 * - convenience defines for the client side code.
620 * - keep this in sync with the readres structure in rdc_prot.h/.x
621 */
622 #define rdcrdresult readres
623 #define rr_status status
624 #define rr_ok readres_u.reply
625 #define rr_bufsize rr_ok.data.data_len
626 #define rr_data rr_ok.data.data_val
627
628 /*
629 * Flags for remote read rpc
630 *
631 * _START must be a unique rpc, _DATA and _END may be OR-d together.
632 */
633 #define RDC_RREAD_DATA 0x1 /* Intermediate rpc with data payload */
634 #define RDC_RREAD_START 0x2 /* Setup rpc */
635 #define RDC_RREAD_END 0x4 /* End rpc */
636 #define RDC_RREAD_FAIL 0x8 /* Primary is failed */
637
638 /*
639 * Flags for remote write rpc
640 */
641 #define RDC_RWRITE_FAIL 0x8 /* Primary is failed */
642
643 /*
644 * macro used to determine if the incomming sq, with sequence
645 * value x, should be placed before the sq with sequence value y.
646 * This has to account for integer wrap. We account for integer
647 * wrap by checking if the difference between x and y is within
648 * half of the maximum integer value (RDC_MAXINT) or not.
649 */
650
651 #define RDC_BITSPERBYTE 8
652 #define RDC_BITS(type) (RDC_BITSPERBYTE * (long)sizeof (type))
653 #define RDC_HIBITI ((unsigned)1 << (RDC_BITS(int) - 1))
654 #define RDC_MAXINT ((int)(~RDC_HIBITI))
655 #define RDC_RANGE ((RDC_MAXINT / 2) -1)
656
657 #define RDC_INFRONT(x, y) (((x < y) && ((y - x) < RDC_RANGE)) ? 1 : \
658 ((x > y) && ((x - y) > RDC_RANGE)) ? 1 : 0)
659
660
661
662
663 #endif /* _KERNEL */
664
665 /*
666 * RDC user-visible information
667 */
668 typedef rdc_set_t rdc_u_info_t;
669
670
671 /*
672 * RDC flags for set state / set cd RPC.
673 * Must remain compatible with rdc RPC protocol version v3.
674 */
675 #define CCIO_NONE 0x0000
676 #define CCIO_ENABLE 0x0008
677 #define CCIO_SLAVE 0x0010
678 #define CCIO_DONE 0x0020
679 #define CCIO_ENABLELOG 0x0100
680 #define CCIO_RSYNC 0x0400
681 #define CCIO_REMOTE 0x2000
682
683
684 /*
685 * In kernel type flags (krdc->type_flag).
686 */
687 #define RDC_CONFIGURED 0x1
688 #define RDC_DISABLEPEND 0x2 /* Suspend/Disable is in progress */
689 #define RDC_ASYNCMODE 0x4
690 #define RDC_RESUMEPEND 0x8
691 #define RDC_RESPONSIBLE 0x10
692 #define RDC_BUSYWAIT 0x20
693 #define RDC_UNREGISTER 0x40 /* Unregister is in progress */
694 #define RDC_QDISABLEPEND 0x100 /* Q Suspend/Disable is in progress */
695
696 #define IS_ENABLED(urdc) ((IS_CONFIGURED(&rdc_k_info[(urdc)->index]) && \
697 (rdc_get_vflags(urdc) & RDC_ENABLED)))
698 #define IS_CONFIGURED(krdc) ((krdc)->type_flag & RDC_CONFIGURED)
699 #define IS_MANY(krdc) ((krdc)->many_next != (krdc))
700 #define IS_MULTI(krdc) ((krdc)->multi_next != NULL)
701
702 #define IS_VALID_INDEX(index) ((index) >= 0 && (index) < rdc_max_sets && \
703 IS_CONFIGURED(&rdc_k_info[(index)]))
704
705 #define RDC_NOFLUSH 0 /* Do not do a flush when starting logging */
706 #define RDC_NOREMOTE 0 /* Do no remote logging notifications */
707 #define RDC_FLUSH 1 /* Do a flush when starting logging */
708 #define RDC_ALLREMOTE 2 /* Notify all remote group members */
709 #define RDC_OTHERREMOTE 4 /* Notify all remote group members except */
710 /* the one corresponding to the current set, */
711 /* to prevent recursion in the case where */
712 /* the request was initiated from the remote */
713 /* node. */
714 #define RDC_FORCE_GROUP 8 /* set all group memebers logging regardless */
715
716 #ifdef _KERNEL
717
718 /*
719 * Functions, vars
720 */
721
722 #define RDC_SYNC_EVENT_TIMEOUT (60 * HZ)
723 typedef struct {
724 clock_t lbolt;
725 int event;
726 int ack;
727 int daemon_waiting; /* Daemon waiting in ioctl */
728 int kernel_waiting; /* Kernel waiting for daemon to reply */
729 char master[NSC_MAXPATH];
730 char group[NSC_MAXPATH];
731 kmutex_t mutex;
732 kcondvar_t cv;
733 kcondvar_t done_cv;
734 } rdc_sync_event_t;
735 extern rdc_sync_event_t rdc_sync_event;
736 extern clock_t rdc_sync_event_timeout;
737 extern kmutex_t rdc_sync_mutex;
738
739 extern rdc_u_info_t *rdc_u_info;
740 extern rdc_k_info_t *rdc_k_info;
741
742 extern int rdc_max_sets;
743
744 extern unsigned long rdc_async_timeout;
745
746 extern int rdc_self_host();
747 extern uint64_t mirror_getsize(int index);
748 extern void rdc_sleepqdiscard(rdc_group_t *);
749
750
751 #ifdef DEBUG
752 extern void rdc_stallzero(int);
753 #endif
754
755 struct rdc_net_dataitem {
756 void *dptr;
757 int len; /* byte count */
758 int mlen; /* actual malloced size */
759 struct rdc_net_dataitem *next;
760 };
761 typedef struct rdc_net_dataitem rdc_net_dataitem_t;
762
763 struct rdc_net_dataset {
764 int id;
765 int inuse;
766 int delpend;
767 int nitems;
768 nsc_off_t pos;
769 nsc_size_t fbalen;
770 rdc_net_dataitem_t *head;
771 rdc_net_dataitem_t *tail;
772 struct rdc_net_dataset *next;
773 };
774 typedef struct rdc_net_dataset rdc_net_dataset_t;
775
776
777 #endif /* _KERNEL */
778
779
780 #define RDC_TCP_DEV "/dev/tcp"
781
782 #define RDC_VERS_MIN RDC_VERSION5
783 #define RDC_VERS_MAX RDC_VERSION7
784
785 #define RDC_HEALTH_THRESHOLD 20
786 #define RDC_MIN_HEALTH_THRES 5
787 #define SNDR_MAXTHREADS 16
788 /*
789 * These next two defines are the default value of the async queue size
790 * They have been calculated to be 8MB of data with an average of
791 * 2K IO size
792 */
793 #define RDC_MAXTHRES_QUEUE 16384 /* max # of fbas on async q */
794 #define RDC_MAX_QITEMS 4096 /* max # of items on async q */
795 #define RDC_ASYNCTHR 2 /* number of async threads */
796
797 #define RDC_RPC_MAX (RDC_MAXDATA + sizeof (net_data5) +\
798 (RPC_MAXDATASIZE - 8192))
799 #define ATM_NONE 0
800 #define ATM_INIT 1
801 #define ATM_EXIT 2
802
803 #define RDC_CLNT_TMOUT 16
804
805 #define BMAP_BLKSIZE 1024
806 #define BMAP_BLKSIZEV7 RDC_MAXDATA
807
808 /* right now we can only trace 1m or less writes to the bitmap (32 bits wide) */
809 #define RDC_MAX_MAXFBAS 2048
810
811 #if defined(_KERNEL)
812 /* kstat interface */
813
814 /*
815 * Per module kstats
816 * only one instance
817 */
818 typedef struct {
819 kstat_named_t m_maxsets; /* Max # of sndr sets */
820 kstat_named_t m_maxfbas; /* Max # of FBAS from nsctl */
821 kstat_named_t m_rpc_timeout; /* global RPC timeout */
822 kstat_named_t m_health_thres; /* Health thread timeout */
823 kstat_named_t m_bitmap_writes; /* True for bitmap writes */
824 kstat_named_t m_clnt_cots_calls; /* # of clnt COTS calls */
825 kstat_named_t m_clnt_clts_calls; /* # of clnt CLTS calls */
826 kstat_named_t m_svc_cots_calls; /* # of server COTS calls */
827 kstat_named_t m_svc_clts_calls; /* # of server CLTS calls */
828 kstat_named_t m_bitmap_ref_delay; /* # of bitmap ref overflows */
829 } sndr_m_stats_t;
830
831 /*
832 * Per set kstats
833 * one instance per configured set
834 */
835 typedef struct {
836 kstat_named_t s_flags; /* from rdc_set_t */
837 kstat_named_t s_syncflags; /* from rdc_set_t */
838 kstat_named_t s_bmpflags; /* from rdc_set_t */
839 kstat_named_t s_syncpos; /* from rdc_set_t */
840 kstat_named_t s_volsize; /* from rdc_set_t */
841 kstat_named_t s_bits_set; /* from rdc_set_t */
842 kstat_named_t s_autosync; /* from rdc_set_t */
843 kstat_named_t s_maxqfbas; /* from rdc_set_t */
844 kstat_named_t s_maxqitems; /* from rdc_set_t */
845 kstat_named_t s_primary_vol; /* from rdc_set_t */
846 kstat_named_t s_secondary_vol; /* from rdc_set_t */
847 kstat_named_t s_bitmap; /* from rdc_set_t */
848 kstat_named_t s_primary_intf; /* from rdc_set_t */
849 kstat_named_t s_secondary_intf; /* from rdc_set_t */
850 kstat_named_t s_type_flag; /* from rdc_k_info_t */
851 kstat_named_t s_bitmap_size; /* from rdc_k_info_t */
852 kstat_named_t s_disk_status; /* from rdc_k_info_t */
853 kstat_named_t s_if_if_down; /* from rdc_if_t */
854 kstat_named_t s_if_rpc_version; /* from rdc_if_t */
855 kstat_named_t s_aqueue_blk_hwm; /* from rdc_k_info_t */
856 kstat_named_t s_aqueue_itm_hwm; /* from rdc_k_info_t */
857 kstat_named_t s_aqueue_throttle; /* from rdc_k_info_t */
858 kstat_named_t s_aqueue_items;
859 kstat_named_t s_aqueue_blocks;
860 kstat_named_t s_aqueue_type;
861 } rdc_info_stats_t;
862 #endif /* _KERNEL */
863
864 #ifndef _SunOS_5_6 /* i.e. 2.7+ */
865 typedef int xdr_t;
866 #else /* i.e. 2.6- */
867 typedef unsigned long rpcprog_t;
868 typedef unsigned long rpcvers_t;
869 typedef unsigned long rpcproc_t;
870 typedef unsigned long rpcprot_t;
871 typedef unsigned long rpcport_t;
872 #endif /* _SunOS_5_6 */
873
874
875 #ifdef _KERNEL
876
877 extern nsc_size_t MAX_RDC_FBAS;
878 extern volatile int net_exit;
879 extern nsc_size_t rdc_maxthres_queue; /* max # of fbas on async q */
880 extern int rdc_max_qitems; /* max # of items on async q */
881 extern int rdc_asyncthr; /* # of async threads */
882
883 #ifdef DEBUG
884 extern kmutex_t rdc_cntlock;
885 extern int rdc_datasetcnt;
886 #endif
887
888 /*
889 * Macro to keep tabs on dataset memory usage.
890 */
891 #ifdef DEBUG
892 #define RDC_DSMEMUSE(x) \
893 mutex_enter(&rdc_cntlock);\
894 rdc_datasetcnt += (x);\
895 mutex_exit(&rdc_cntlock);
896 #else
897 #define RDC_DSMEMUSE(x)
898 #endif
899
900
901
902
903
904 extern kmutex_t rdc_ping_lock;
905 extern rdc_if_t *rdc_if_top;
906
907 extern int _rdc_enqueue_write(rdc_k_info_t *, nsc_off_t, nsc_size_t, int,
908 nsc_buf_t *);
909 extern int rdc_net_state(int, int);
910 extern int rdc_net_getbmap(int, int);
911 extern int rdc_net_getsize(int, uint64_t *);
912 extern int rdc_net_write(int, int, nsc_buf_t *, nsc_off_t, nsc_size_t, uint_t,
913 int, netwriteres *);
914 extern int rdc_net_read(int, int, nsc_buf_t *, nsc_off_t, nsc_size_t);
915 extern int _rdc_remote_read(rdc_k_info_t *, nsc_buf_t *, nsc_off_t, nsc_size_t,
916 int);
917 extern int _rdc_multi_write(nsc_buf_t *, nsc_off_t, nsc_size_t, int,
918 rdc_k_info_t *);
919 extern int rdc_start_server(struct rdc_svc_args *, int);
920 extern aio_buf_t *rdc_aio_buf_get(rdc_buf_t *, int);
921 extern void rdc_aio_buf_del(rdc_buf_t *, rdc_k_info_t *);
922 extern aio_buf_t *rdc_aio_buf_add(int, rdc_buf_t *);
923 extern int rdc_net_getstate(rdc_k_info_t *, int *, int *, int *, int);
924 extern kmutex_t rdc_conf_lock;
925 extern kmutex_t rdc_many_lock;
926 extern int rdc_drain_queue(int);
927 extern int flush_group_queue(int);
928 extern void rdc_dev_close(rdc_k_info_t *);
929 extern int rdc_dev_open(rdc_set_t *, int);
930 extern void rdc_get_details(rdc_k_info_t *);
931 extern int rdc_lookup_bitmap(char *);
932 extern int rdc_lookup_enabled(char *, int);
933 extern int rdc_lookup_byaddr(rdc_set_t *);
934 extern int rdc_lookup_byname(rdc_set_t *);
935 extern int rdc_intercept(rdc_k_info_t *);
936 extern int rdc_unintercept(rdc_k_info_t *);
937 extern int _rdc_rsrv_devs(rdc_k_info_t *, int, int);
938 extern void _rdc_rlse_devs(rdc_k_info_t *, int);
939 extern void _rdc_unload(void);
940 extern int _rdc_load(void);
941 extern int _rdc_configure(void);
942 extern void _rdc_deconfigure(void);
943 extern void _rdc_async_throttle(rdc_k_info_t *, long);
944 extern int rdc_writer(int);
945 extern int rdc_dump_alloc_bufs_cd(int);
946 extern void rdc_dump_alloc_bufs(rdc_if_t *);
947 extern int rdc_check_secondary(rdc_if_t *, int);
948 extern void rdc_dump_queue(int);
949 extern int rdc_isactive_if(struct netbuf *, struct netbuf *);
950 extern rdc_if_t *rdc_add_to_if(rdc_srv_t *, struct netbuf *, struct netbuf *,
951 int);
952 extern void rdc_remove_from_if(rdc_if_t *);
953 extern void rdc_set_if_vers(rdc_u_info_t *, rpcvers_t);
954
955 extern void rdc_print_svinfo(rdc_srv_t *, char *);
956 extern rdc_srv_t *rdc_create_svinfo(char *, struct netbuf *,
957 struct knetconfig *);
958 extern void rdc_destroy_svinfo(rdc_srv_t *);
959
960 extern void init_rdc_netbuf(struct netbuf *);
961 extern void free_rdc_netbuf(struct netbuf *);
962 extern void dup_rdc_netbuf(const struct netbuf *, struct netbuf *);
963 extern int rdc_netbuf_toint(struct netbuf *);
964 extern struct netbuf *rdc_int_tonetbuf(int);
965 extern void rdc_lor(const uchar_t *, uchar_t *, int);
966 extern int rdc_resume2(rdc_k_info_t *);
967 extern void rdc_set_flags(rdc_u_info_t *, int);
968 extern void rdc_clr_flags(rdc_u_info_t *, int);
969 extern int rdc_get_vflags(rdc_u_info_t *);
970 extern void rdc_set_mflags(rdc_u_info_t *, int);
971 extern void rdc_clr_mflags(rdc_u_info_t *, int);
972 extern int rdc_get_mflags(rdc_u_info_t *);
973 extern void rdc_set_flags_log(rdc_u_info_t *, int, char *);
974 extern void rdc_group_log(rdc_k_info_t *krdc, int flush, char *why);
975 extern int _rdc_config(void *, int, spcs_s_info_t, int *);
976 extern void rdc_many_enter(rdc_k_info_t *);
977 extern void rdc_many_exit(rdc_k_info_t *);
978 extern void rdc_group_enter(rdc_k_info_t *);
979 extern void rdc_group_exit(rdc_k_info_t *);
980 extern int _rdc_sync_event_wait(void *, void *, int, spcs_s_info_t, int *);
981 extern int _rdc_sync_event_notify(int, char *, char *);
982 extern int _rdc_link_down(void *, int, spcs_s_info_t, int *);
983 extern void rdc_delgroup(rdc_group_t *);
984 extern int rdc_write_bitmap_fba(rdc_k_info_t *, nsc_off_t);
985 extern int rdc_bitmapset(int, char *, char *, void *, int, nsc_off_t, int);
986 extern rdc_net_dataset_t *rdc_net_add_set(int);
987 extern rdc_net_dataset_t *rdc_net_get_set(int, int);
988 extern void rdc_net_put_set(int, rdc_net_dataset_t *);
989 extern void rdc_net_del_set(int, rdc_net_dataset_t *);
990 extern void rdc_net_free_set(rdc_k_info_t *, rdc_net_dataset_t *);
991 extern int rdc_lookup_byhostdev(char *intf, char *file);
992 extern int rdc_lookup_configured(char *path);
993 extern void rdc_dump_dsets(int);
994 extern void set_busy(rdc_k_info_t *);
995 extern void wakeup_busy(rdc_k_info_t *);
996
997
998 #ifdef DEBUG
999 extern int rdc_async6(void *, int mode, int *);
1000 extern int rdc_readgen(void *, int, int *);
1001 #endif
1002
1003 #endif /* _KERNEL */
1004
1005 #ifdef __cplusplus
1006 }
1007 #endif
1008
1009 #endif /* _RDC_IO_H */