1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 /*
28 * RDC interface health monitoring code.
29 */
30
31 #include <sys/types.h>
32 #include <sys/ksynch.h>
33 #include <sys/errno.h>
34 #include <sys/debug.h>
35 #include <sys/cmn_err.h>
36 #include <sys/kmem.h>
37
38 #include <sys/errno.h>
39
40 #ifdef _SunOS_2_6
41 /*
42 * on 2.6 both dki_lock.h and rpc/types.h define bool_t so we
43 * define enum_t here as it is all we need from rpc/types.h
44 * anyway and make it look like we included it. Yuck.
45 */
46 #define _RPC_TYPES_H
47 typedef int enum_t;
48 #else
49 #ifndef DS_DDICT
50 #include <rpc/types.h>
51 #endif
52 #endif /* _SunOS_2_6 */
53
54 #include <sys/ddi.h>
55 #include <sys/nsc_thread.h>
56 #ifdef DS_DDICT
57 #include <sys/nsctl/contract.h>
58 #endif
59 #include <sys/nsctl/nsctl.h>
60
61 #include <sys/unistat/spcs_s.h>
62 #include <sys/unistat/spcs_s_k.h>
63 #include <sys/unistat/spcs_errors.h>
64
65 #include "rdc_io.h"
66 #include "rdc_clnt.h"
67
68
69 /*
70 * Forward declarations.
71 */
72
73 static void rdc_update_health(rdc_if_t *);
74
75 /*
76 * Global data.
77 */
78
79 /*
80 * These structures are added when a new host name is introduced to the
81 * kernel. They never disappear (but that won't waste much space at all).
82 */
83 typedef struct rdc_link_down {
84 char host[MAX_RDC_HOST_SIZE]; /* The host name of this link */
85 int waiting; /* A user is waiting to be woken up */
86 int link_down; /* The current state of the link */
87 struct rdc_link_down *next; /* Chain */
88 kcondvar_t syncd_cv; /* Syncd wakeup */
89 kmutex_t syncd_mutex; /* Lock for syncd_cv */
90 } rdc_link_down_t;
91 static rdc_link_down_t *rdc_link_down = NULL;
92
93 int rdc_health_thres = RDC_HEALTH_THRESHOLD;
94 rdc_if_t *rdc_if_top;
95
96
97 /*
98 * IPv6 addresses are represented as 16bit hexadecimal integers
99 * separated by colons. Contiguous runs of zeros can be abbreviated by
100 * double colons:
101 * FF02:0:0:0:0:1:200E:8C6C
102 * |
103 * v
104 * FF02::1:200E:8C6C
105 */
106 void
107 rdc_if_ipv6(const uint16_t *addr, char *buf)
108 {
109 const int end = 8; /* 8 shorts, 128 bits in an IPv6 address */
110 int i;
111
112 for (i = 0; i < end; i++) {
113 if (i > 0)
114 (void) sprintf(buf, "%s:", buf);
115
116 if (addr[i] != 0 || i == 0 || i == (end - 1)) {
117 /* first, last, or non-zero value */
118 (void) sprintf(buf, "%s%x", buf, (int)addr[i]);
119 } else {
120 if ((i + 1) < end && addr[i + 1] != 0) {
121 /* single zero */
122 (void) sprintf(buf, "%s%x", buf, (int)addr[i]);
123 } else {
124 /* skip contiguous zeros */
125 while ((i + 1) < end && addr[i + 1] == 0)
126 i++;
127 }
128 }
129 }
130 }
131
132 static void
133 rdc_if_xxx(rdc_if_t *ip, char *updown)
134 {
135 if (strcmp("inet6", ip->srv->ri_knconf->knc_protofmly) == 0) {
136 uint16_t *this = (uint16_t *)ip->ifaddr.buf;
137 uint16_t *other = (uint16_t *)ip->r_ifaddr.buf;
138 char this_str[256], other_str[256];
139
140 bzero(this_str, sizeof (this_str));
141 bzero(other_str, sizeof (other_str));
142 rdc_if_ipv6(&this[4], this_str);
143 rdc_if_ipv6(&other[4], other_str);
144
145 cmn_err(CE_NOTE, "!SNDR: Interface %s <==> %s : %s",
146 this_str, other_str, updown);
147 } else {
148 uchar_t *this = (uchar_t *)ip->ifaddr.buf;
149 uchar_t *other = (uchar_t *)ip->r_ifaddr.buf;
150
151 cmn_err(CE_NOTE,
152 "!SNDR: Interface %d.%d.%d.%d <==> %d.%d.%d.%d : %s",
153 (int)this[4], (int)this[5], (int)this[6], (int)this[7],
154 (int)other[4], (int)other[5], (int)other[6], (int)other[7],
155 updown);
156 }
157 }
158
159
160 static void
161 rdc_if_down(rdc_if_t *ip)
162 {
163 rdc_if_xxx(ip, "Down");
164 }
165
166
167 static void
168 rdc_if_up(rdc_if_t *ip)
169 {
170 rdc_if_xxx(ip, "Up");
171 }
172
173
174 /*
175 * Health monitor for a single interface.
176 *
177 * The secondary sends ping RPCs to the primary.
178 * The primary just stores the results and updates its structures.
179 */
180 static void
181 rdc_health_thread(void *arg)
182 {
183 rdc_if_t *ip = (rdc_if_t *)arg;
184 struct rdc_ping ping;
185 struct rdc_ping6 ping6;
186 struct timeval t;
187 int down = 1;
188 int ret, err;
189 int sec = 0;
190 char ifaddr[RDC_MAXADDR];
191 char r_ifaddr[RDC_MAXADDR];
192 uint16_t *sp;
193
194 bcopy(ip->ifaddr.buf, ifaddr, ip->ifaddr.len);
195 sp = (uint16_t *)ifaddr;
196 *sp = htons(*sp);
197 bcopy(ip->r_ifaddr.buf, r_ifaddr, ip->r_ifaddr.len);
198 sp = (uint16_t *)r_ifaddr;
199 *sp = htons(*sp);
200
201 while ((ip->exiting != 1) && (net_exit != ATM_EXIT)) {
202 delay(HZ);
203
204 /* setup RPC timeout */
205
206 t.tv_sec = rdc_rpc_tmout;
207 t.tv_usec = 0;
208
209 if (ip->issecondary && !ip->no_ping) {
210 if (ip->rpc_version < RDC_VERSION7) {
211 bcopy(ip->r_ifaddr.buf, ping6.p_ifaddr,
212 RDC_MAXADDR);
213 /* primary ifaddr */
214 bcopy(ip->ifaddr.buf, ping6.s_ifaddr,
215 RDC_MAXADDR);
216 /* secondary ifaddr */
217 err = rdc_clnt_call_any(ip->srv, ip,
218 RDCPROC_PING4, xdr_rdc_ping6,
219 (char *)&ping6, xdr_int, (char *)&ret, &t);
220 } else {
221 ping.p_ifaddr.buf = r_ifaddr;
222 ping.p_ifaddr.len = ip->r_ifaddr.len;
223 ping.p_ifaddr.maxlen = ip->r_ifaddr.len;
224 ping.s_ifaddr.buf = ifaddr;
225 ping.s_ifaddr.len = ip->ifaddr.len;
226 ping.s_ifaddr.maxlen = ip->ifaddr.len;
227 err = rdc_clnt_call_any(ip->srv, ip,
228 RDCPROC_PING4, xdr_rdc_ping, (char *)&ping,
229 xdr_int, (char *)&ret, &t);
230 }
231
232
233 if (err || ret) {
234 /* RPC failed - link is down */
235 if (!down && !ip->isprimary) {
236 /*
237 * don't print messages if also
238 * a primary - the primary will
239 * take care of it.
240 */
241 rdc_if_down(ip);
242 down = 1;
243 }
244 rdc_dump_alloc_bufs(ip);
245 ip->no_ping = 1;
246
247 /*
248 * Start back at the max possible version
249 * since the remote server could come back
250 * on a different protocol version.
251 */
252 mutex_enter(&rdc_ping_lock);
253 ip->rpc_version = RDC_VERS_MAX;
254 mutex_exit(&rdc_ping_lock);
255 } else {
256 if (down && !ip->isprimary) {
257 /*
258 * was failed, but now ok
259 *
260 * don't print messages if also
261 * a primary - the primary will
262 * take care of it.
263 */
264 rdc_if_up(ip);
265 down = 0;
266 }
267 }
268 }
269 if (!ip->isprimary && down && ++sec == 5) {
270 sec = 0;
271 rdc_dump_alloc_bufs(ip);
272 }
273
274 if (ip->isprimary)
275 rdc_update_health(ip);
276 }
277
278 /* signal that this thread is done */
279 ip->exiting = 2;
280 }
281
282
283 int
284 rdc_isactive_if(struct netbuf *addr, struct netbuf *r_addr)
285 {
286 rdc_if_t *ip;
287 int rc = 0;
288
289 /* search for existing interface structure */
290
291 mutex_enter(&rdc_ping_lock);
292 for (ip = rdc_if_top; ip; ip = ip->next) {
293 if (ip->exiting != 0)
294 continue;
295 if (((bcmp(ip->ifaddr.buf, addr->buf, addr->len) == 0) &&
296 (bcmp(ip->r_ifaddr.buf, r_addr->buf, r_addr->len) == 0)) ||
297 ((bcmp(ip->r_ifaddr.buf, addr->buf, addr->len) == 0) &&
298 (bcmp(ip->ifaddr.buf, r_addr->buf, r_addr->len) == 0))) {
299 /* found matching interface structure */
300 if (ip->isprimary && !ip->if_down) {
301 rc = 1;
302 } else if (ip->issecondary && !ip->no_ping) {
303 rc = 1;
304 }
305 break;
306 }
307 }
308 mutex_exit(&rdc_ping_lock);
309 return (rc);
310 }
311
312 /*
313 * Set the rdc rpc version of the rdc_if_t.
314 *
315 * Called from incoming rpc calls which start before
316 * the health service becomes established.
317 */
318 void
319 rdc_set_if_vers(rdc_u_info_t *urdc, rpcvers_t vers)
320 {
321 rdc_if_t *ip;
322 struct netbuf *addr, *r_addr;
323
324 if (rdc_get_vflags(urdc) & RDC_PRIMARY) {
325 addr = &(urdc->primary.addr);
326 r_addr = &(urdc->secondary.addr);
327 } else {
328 addr = &(urdc->secondary.addr);
329 r_addr = &(urdc->primary.addr);
330 }
331
332 /* search for existing interface structure */
333
334 mutex_enter(&rdc_ping_lock);
335 for (ip = rdc_if_top; ip; ip = ip->next) {
336 if (ip->exiting != 0)
337 continue;
338 if (((bcmp(ip->ifaddr.buf, addr->buf, addr->len) == 0) &&
339 (bcmp(ip->r_ifaddr.buf, r_addr->buf, r_addr->len) == 0)) ||
340 ((bcmp(ip->r_ifaddr.buf, addr->buf, addr->len) == 0) &&
341 (bcmp(ip->ifaddr.buf, r_addr->buf, r_addr->len) == 0))) {
342 /* found matching interface structure */
343 ip->rpc_version = vers;
344 #ifdef DEBUG
345 cmn_err(CE_NOTE, "!rdc intf %p rpc version set to %u",
346 (void *)ip, vers);
347 #endif
348 break;
349 }
350 }
351 mutex_exit(&rdc_ping_lock);
352 }
353
354 /*
355 * Free all the rdc_link_down structures (only at module unload time)
356 */
357 void
358 rdc_link_down_free()
359 {
360 rdc_link_down_t *p;
361 rdc_link_down_t *q;
362
363 if (rdc_link_down == NULL)
364 return;
365
366 for (p = rdc_link_down->next; p != rdc_link_down; ) {
367 q = p;
368 p = p->next;
369 kmem_free(q, sizeof (*q));
370 }
371 kmem_free(rdc_link_down, sizeof (*q));
372 rdc_link_down = NULL;
373 }
374
375
376 /*
377 * Look up the supplied hostname in the rdc_link_down chain. Add a new
378 * entry if it isn't found. Return a pointer to the new or found entry.
379 */
380 static rdc_link_down_t *
381 rdc_lookup_host(char *host)
382 {
383 rdc_link_down_t *p;
384
385 mutex_enter(&rdc_ping_lock);
386
387 if (rdc_link_down == NULL) {
388 rdc_link_down = kmem_zalloc(sizeof (*rdc_link_down), KM_SLEEP);
389 rdc_link_down->next = rdc_link_down;
390 }
391
392 for (p = rdc_link_down->next; p != rdc_link_down; p = p->next) {
393 if (strcmp(host, p->host) == 0) {
394 /* Match */
395 mutex_exit(&rdc_ping_lock);
396 return (p);
397 }
398 }
399
400 /* No match, must create a new entry */
401
402 p = kmem_zalloc(sizeof (*p), KM_SLEEP);
403 p->link_down = 1;
404 p->next = rdc_link_down->next;
405 rdc_link_down->next = p;
406 (void) strncpy(p->host, host, MAX_RDC_HOST_SIZE);
407 mutex_init(&p->syncd_mutex, NULL, MUTEX_DRIVER, NULL);
408 cv_init(&p->syncd_cv, NULL, CV_DRIVER, NULL);
409
410 mutex_exit(&rdc_ping_lock);
411 return (p);
412 }
413
414
415 /*
416 * Handle the RDC_LINK_DOWN ioctl.
417 * The user specifies which host they're interested in.
418 * This function is woken up when the link to that host goes down.
419 */
420
421 /* ARGSUSED3 */
422 int
423 _rdc_link_down(void *arg, int mode, spcs_s_info_t kstatus, int *rvp)
424 {
425 char host[MAX_RDC_HOST_SIZE];
426 rdc_link_down_t *syncdp;
427 clock_t timeout = RDC_SYNC_EVENT_TIMEOUT * 2; /* 2 min */
428 int rc = 0;
429
430 if (ddi_copyin(arg, host, MAX_RDC_HOST_SIZE, mode))
431 return (EFAULT);
432
433
434 syncdp = rdc_lookup_host(host);
435
436 mutex_enter(&syncdp->syncd_mutex);
437 if (!syncdp->link_down) {
438 syncdp->waiting = 1;
439 if (cv_timedwait_sig(&syncdp->syncd_cv, &syncdp->syncd_mutex,
440 nsc_lbolt() + timeout) == 0) {
441 /* Woken by a signal, not a link down event */
442 syncdp->waiting = 0;
443 rc = EAGAIN;
444 spcs_s_add(kstatus, rc);
445 }
446
447 }
448 mutex_exit(&syncdp->syncd_mutex);
449
450 return (rc);
451 }
452
453
454 /*
455 * Add an RDC set to an interface
456 *
457 * If the interface is new, add it to the list of interfaces.
458 */
459 rdc_if_t *
460 rdc_add_to_if(rdc_srv_t *svp, struct netbuf *addr, struct netbuf *r_addr,
461 int primary)
462 {
463 rdc_if_t *new, *ip;
464
465 if ((addr->buf == NULL) || (r_addr->buf == NULL))
466 return (NULL);
467
468 /* setup a new interface structure */
469 new = (rdc_if_t *)kmem_zalloc(sizeof (*new), KM_SLEEP);
470 if (!new)
471 return (NULL);
472
473 dup_rdc_netbuf(addr, &new->ifaddr);
474 dup_rdc_netbuf(r_addr, &new->r_ifaddr);
475 new->rpc_version = RDC_VERS_MAX;
476 new->srv = rdc_create_svinfo(svp->ri_hostname, &svp->ri_addr,
477 svp->ri_knconf);
478 new->old_pulse = -1;
479 new->new_pulse = 0;
480
481 if (!new->srv) {
482 free_rdc_netbuf(&new->r_ifaddr);
483 free_rdc_netbuf(&new->ifaddr);
484 kmem_free(new, sizeof (*new));
485 return (NULL);
486 }
487
488 /* search for existing interface structure */
489
490 mutex_enter(&rdc_ping_lock);
491
492 for (ip = rdc_if_top; ip; ip = ip->next) {
493 if ((bcmp(ip->ifaddr.buf, addr->buf, addr->len) == 0) &&
494 (bcmp(ip->r_ifaddr.buf, r_addr->buf, r_addr->len) == 0) &&
495 ip->exiting == 0) {
496 /* found matching interface structure */
497 break;
498 }
499 }
500
501 if (!ip) {
502 /* add new into the chain */
503
504 new->next = rdc_if_top;
505 rdc_if_top = new;
506 ip = new;
507
508 /* start daemon */
509
510 ip->last = nsc_time();
511 ip->deadness = 1;
512 ip->if_down = 1;
513
514 if (nsc_create_process(rdc_health_thread, ip, TRUE)) {
515 mutex_exit(&rdc_ping_lock);
516 return (NULL);
517 }
518 }
519
520 /* mark usage type */
521
522 if (primary) {
523 ip->isprimary = 1;
524 } else {
525 ip->issecondary = 1;
526 ip->no_ping = 0;
527 }
528
529 mutex_exit(&rdc_ping_lock);
530
531 /* throw away new if it was not used */
532
533 if (ip != new) {
534 free_rdc_netbuf(&new->r_ifaddr);
535 free_rdc_netbuf(&new->ifaddr);
536 rdc_destroy_svinfo(new->srv);
537 kmem_free(new, sizeof (*new));
538 }
539
540 return (ip);
541 }
542
543
544 /*
545 * Update an interface following the removal of an RDC set.
546 *
547 * If there are no more RDC sets using the interface, delete it from
548 * the list of interfaces.
549 *
550 * Either clear krdc->intf, or ensure !IS_CONFIGURED(krdc) before calling this.
551 */
552 void
553 rdc_remove_from_if(rdc_if_t *ip)
554 {
555 rdc_k_info_t *krdc;
556 rdc_u_info_t *urdc;
557 rdc_if_t **ipp;
558 int pfound = 0;
559 int sfound = 0;
560 int delete = 1;
561 int index;
562
563 mutex_enter(&rdc_ping_lock);
564
565 /*
566 * search for RDC sets using this interface and update
567 * the isprimary and issecondary flags.
568 */
569
570 for (index = 0; index < rdc_max_sets; index++) {
571 krdc = &rdc_k_info[index];
572 urdc = &rdc_u_info[index];
573 if (IS_CONFIGURED(krdc) && krdc->intf == ip) {
574 delete = 0;
575
576 if (rdc_get_vflags(urdc) & RDC_PRIMARY) {
577 pfound = 1;
578 } else {
579 sfound = 1;
580 }
581
582 if (pfound && sfound)
583 break;
584 }
585 }
586
587 ip->isprimary = pfound;
588 ip->issecondary = sfound;
589
590 if (!delete || ip->exiting > 0) {
591 mutex_exit(&rdc_ping_lock);
592 return;
593 }
594
595 /* mark and wait for daemon to exit */
596
597 ip->exiting = 1;
598
599 mutex_exit(&rdc_ping_lock);
600
601 while (ip->exiting == 1)
602 delay(drv_usectohz(10));
603
604 mutex_enter(&rdc_ping_lock);
605
606 ASSERT(ip->exiting == 2);
607
608 /* remove from chain */
609
610 for (ipp = &rdc_if_top; *ipp; ipp = &((*ipp)->next)) {
611 if (*ipp == ip) {
612 *ipp = ip->next;
613 break;
614 }
615 }
616
617 mutex_exit(&rdc_ping_lock);
618
619 /* free unused interface structure */
620
621 free_rdc_netbuf(&ip->r_ifaddr);
622 free_rdc_netbuf(&ip->ifaddr);
623 rdc_destroy_svinfo(ip->srv);
624 kmem_free(ip, sizeof (*ip));
625 }
626
627
628 /*
629 * Check the status of the link to the secondary, and optionally update
630 * the primary-side ping variables.
631 *
632 * For use on a primary only.
633 *
634 * Returns:
635 * TRUE - interface up.
636 * FALSE - interface down.
637 */
638 int
639 rdc_check_secondary(rdc_if_t *ip, int update)
640 {
641 int rc = TRUE;
642
643 if (!ip || !ip->isprimary) {
644 #ifdef DEBUG
645 cmn_err(CE_WARN,
646 "!rdc_check_secondary: ip %p, isprimary %d, issecondary %d",
647 (void *) ip, ip ? ip->isprimary : 0,
648 ip ? ip->issecondary : 0);
649 #endif
650 return (FALSE);
651 }
652
653 if (!ip->deadness) {
654 #ifdef DEBUG
655 cmn_err(CE_WARN, "!rdc_check_secondary: ip %p, ip->deadness %d",
656 (void *) ip, ip->deadness);
657 #endif
658 return (FALSE);
659 }
660
661 if (!update) {
662 /* quick look */
663 return ((ip->deadness > rdc_health_thres) ? FALSE : TRUE);
664 }
665
666 /* update (slow) with lock */
667
668 mutex_enter(&rdc_ping_lock);
669
670 if (ip->old_pulse == ip->new_pulse) {
671 /*
672 * ping has not been received since last update
673 * or we have not yet been pinged,
674 * the health thread has started only as a
675 * local client so far, not so on the other side
676 */
677
678 if (ip->last != nsc_time()) {
679 /* time has passed, so move closer to death */
680
681 ip->last = nsc_time();
682 ip->deadness++;
683
684 if (ip->deadness <= 0) {
685 /* avoid the wrap */
686 ip->deadness = rdc_health_thres + 1;
687 }
688 }
689
690 if (ip->deadness > rdc_health_thres) {
691 rc = FALSE;
692 /*
693 * Start back at the max possible version
694 * since the remote server could come back
695 * on a different protocol version.
696 */
697 ip->rpc_version = RDC_VERS_MAX;
698 }
699 } else {
700 ip->old_pulse = ip->new_pulse;
701 }
702
703 mutex_exit(&rdc_ping_lock);
704 return (rc);
705 }
706
707
708 /*
709 * Update the interface structure with the latest ping info, and
710 * perform interface up/down transitions if required.
711 *
712 * For use on a primary only.
713 */
714 static void
715 rdc_update_health(rdc_if_t *ip)
716 {
717 rdc_k_info_t *krdc;
718 rdc_u_info_t *urdc;
719 int index;
720 rdc_link_down_t *syncdp;
721
722 if (!ip->isprimary) {
723 #ifdef DEBUG
724 cmn_err(CE_WARN,
725 "!rdc_update_health: ip %p, isprimary %d, issecondary %d",
726 (void *) ip, ip ? ip->isprimary : 0,
727 ip ? ip->issecondary : 0);
728 #endif
729 return;
730 }
731
732 if (!rdc_check_secondary(ip, TRUE)) {
733 /* interface down */
734 if (!ip->if_down) {
735 rdc_if_down(ip);
736 ip->if_down = 1;
737
738 /* scan rdc sets and update status */
739
740 for (index = 0; index < rdc_max_sets; index++) {
741 krdc = &rdc_k_info[index];
742 urdc = &rdc_u_info[index];
743
744 if (IS_ENABLED(urdc) && (krdc->intf == ip) &&
745 (rdc_get_vflags(urdc) & RDC_PRIMARY) &&
746 !(rdc_get_vflags(urdc) & RDC_LOGGING)) {
747 /* mark down */
748
749 rdc_group_enter(krdc);
750 /*
751 * check for possible race with
752 * with delete logic
753 */
754 if (!IS_ENABLED(urdc)) {
755 rdc_group_exit(krdc);
756 continue;
757 }
758 rdc_group_log(krdc, RDC_NOFLUSH |
759 RDC_NOREMOTE | RDC_QUEUING,
760 "hm detected secondary "
761 "interface down");
762
763 rdc_group_exit(krdc);
764
765 /* dump async queues */
766 rdc_dump_queue(index);
767 }
768 }
769
770 /* dump allocated bufs */
771 rdc_dump_alloc_bufs(ip);
772 }
773
774 syncdp = rdc_lookup_host(ip->srv->ri_hostname);
775 mutex_enter(&syncdp->syncd_mutex);
776 if (syncdp->link_down == 0) {
777 /* Link has gone down, notify rdcsyncd daemon */
778 syncdp->link_down = 1;
779 if (syncdp->waiting) {
780 syncdp->waiting = 0;
781 cv_signal(&syncdp->syncd_cv);
782 }
783 }
784 mutex_exit(&syncdp->syncd_mutex);
785 } else {
786 /* interface up */
787 if (ip->if_down && ip->isprimary) {
788 rdc_if_up(ip);
789 ip->if_down = 0;
790 }
791
792 syncdp = rdc_lookup_host(ip->srv->ri_hostname);
793 mutex_enter(&syncdp->syncd_mutex);
794 if (syncdp->link_down) {
795 /* Link has come back up */
796 syncdp->link_down = 0;
797 }
798 mutex_exit(&syncdp->syncd_mutex);
799 }
800 }