1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #include <sys/types.h>
  28 #include <sys/wait.h>
  29 #include <stdio.h>
  30 #include <errno.h>
  31 #include <limits.h>
  32 #include <fcntl.h>
  33 #include <strings.h>
  34 #include <stdlib.h>
  35 #include <unistd.h>
  36 #include <pthread.h>
  37 #include <thread.h>
  38 
  39 #include <locale.h>
  40 #include <langinfo.h>
  41 #include <libintl.h>
  42 #include <stdarg.h>
  43 
  44 #include <sys/nsctl/rdc_io.h>
  45 #include <sys/nsctl/rdc_ioctl.h>
  46 #include <sys/nsctl/rdc_prot.h>
  47 
  48 #include <sys/nsctl/cfg.h>
  49 
  50 #include <sys/unistat/spcs_s.h>
  51 #include <sys/unistat/spcs_s_u.h>
  52 #include <sys/unistat/spcs_errors.h>
  53 
  54 #include <sys/nsctl/librdc.h>
  55 
  56 #include "rdcadm.h"
  57 
  58 
  59 #define RDCADM "/usr/sbin/sndradm"
  60 #define IIADM "/usr/sbin/iiadm"
  61 
  62 #define UPDATE "update"
  63 #define NOUPDATE "noupdate"
  64 
  65 #define RESYNC_SLEEP    (3 * 60)        /* Three minutes */
  66 #define MAIN_SLEEP      (5 * 60)        /* Five minutes */
  67 #define CFG_WAIT_SLEEP  (5)             /* 5 sec */
  68 
  69 #define MAXHOSTS 1024
  70 mutex_t cfglock = DEFAULTMUTEX;
  71 #define LOCKCFG() (void) mutex_lock(&cfglock);
  72 #define UNLOCKCFG() (void) mutex_unlock(&cfglock);
  73 
  74 typedef struct host_list_s {
  75         char *hosts[MAXHOSTS];
  76         int numhosts;
  77         int configured[MAXHOSTS];
  78         mutex_t hosts_mutex;
  79 } host_list_t;
  80 
  81 host_list_t *host_list;
  82 
  83 extern char *basename(char *);
  84 int rdc_maxsets;
  85 char *program;
  86 
  87 static int clustered = 0;
  88 
  89 int isnewhost(char *host);
  90 void *wait_sync_event();
  91 void *wait_link_down(void *host);
  92 void rdc_sync(char *tohost);
  93 void remove_from_hostlist(char *host);
  94 void sync_start(char *master);
  95 void sync_complete(char *master);
  96 void cleanup_hostlist();
  97 void group_start(char *group);
  98 void group_complete(char *group);
  99 
 100 
 101 void
 102 init_host_list(void)
 103 {
 104         host_list = calloc(1, sizeof (host_list_t));
 105         if (host_list == NULL) {
 106                 spcs_log("sndr", NULL,
 107                     gettext("host list not initialized, cannot run"));
 108                 rdc_err(NULL, gettext("host list not initialized, cannot run"));
 109         }
 110         (void) mutex_init(&host_list->hosts_mutex, USYNC_THREAD, NULL);
 111 }
 112 
 113 /* ARGSUSED */
 114 #ifdef lint
 115 void
 116 sndrsyncd_lintmain(argc, argv)
 117 #else
 118 int
 119 main(argc, argv)
 120 #endif
 121 int argc;
 122 char **argv;
 123 {
 124         rdc_status_t *rdc_info;
 125         int size;
 126         int i;
 127         pid_t pid;
 128         spcs_s_info_t ustatus;
 129         int rc, trc;
 130         int first = 0;
 131         char *required;
 132 
 133         (void) setlocale(LC_ALL, "");
 134         (void) textdomain("rdc");
 135 
 136         ustatus = spcs_s_ucreate();
 137 
 138         program = basename(argv[0]);
 139 
 140         init_host_list();
 141 
 142         rc = rdc_check_release(&required);
 143         if (rc < 0) {
 144                 rdc_err(NULL,
 145                     gettext("unable to determine the current "
 146                     "Solaris release: %s\n"), strerror(errno));
 147                 /* NOTREACHED */
 148         } else if (rc == FALSE) {
 149                 rdc_err(NULL,
 150                     gettext("incorrect Solaris release (requires %s)\n"),
 151                     required);
 152                 /* NOTREACHED */
 153         }
 154 
 155         clustered = cfg_iscluster();
 156         if (clustered < 0) {
 157                 rdc_err(NULL, gettext("unable to ascertain environment"));
 158         }
 159 
 160         rdc_maxsets = rdc_get_maxsets();
 161         if (rdc_maxsets == -1) {
 162                 spcs_log("sndr", NULL,
 163                     gettext("%s: unable to get maxsets value from kernel"),
 164                     program);
 165                 rdc_err(NULL,
 166                     gettext("unable to get maxsets value from kernel"));
 167         }
 168         size = sizeof (rdc_status_t) + (sizeof (rdc_set_t) * (rdc_maxsets - 1));
 169         rdc_info = malloc(size);
 170         if (rdc_info == NULL) {
 171                 spcs_log("sndr", NULL,
 172                     gettext("%s: unable to allocate %ld bytes"),
 173                     program, size);
 174                 rdc_err(NULL,
 175                         gettext("unable to allocate %ld bytes"), size);
 176         }
 177         bzero(rdc_info, size);
 178 
 179         rdc_info->nset = rdc_maxsets;
 180 
 181         /*
 182          * Fork off a child that becomes the daemon.
 183          */
 184         if ((pid = fork()) > 0)
 185                 exit(0);
 186         else if (pid < 0) {
 187                 spcs_log("sndr", NULL,
 188                     gettext("%s: cannot fork: %s"),
 189                     program, strerror(errno));
 190                 rdc_err(NULL, gettext("cannot fork: %s\n"),
 191                     strerror(errno));
 192         }
 193 
 194         /*
 195          * In child - become daemon.
 196          */
 197 
 198         for (i = 0; i < 3; i++)
 199                 (void) close(i);
 200 
 201         (void) open("/dev/console", O_WRONLY|O_APPEND);
 202         (void) dup(0);
 203         (void) dup(0);
 204         (void) close(0);
 205 
 206         (void) setpgrp();
 207 
 208         (void) setlocale(LC_ALL, "");
 209         (void) textdomain("rdc");
 210 
 211         /* launch a thread to wait for sync start and sync stop events */
 212 
 213         if ((trc = thr_create(NULL, 0, wait_sync_event, NULL,
 214             THR_BOUND|THR_DETACHED, NULL)) != 0) {
 215                 spcs_log("sndr", NULL,
 216                     gettext("%s: unable to create thread wait_sync_event"),
 217                     program);
 218                 rdc_warn(NULL,
 219                     gettext("%s unable to create thread wait_sync_event"),
 220                     program);
 221         } else {
 222 #ifdef DEBUG
 223                 spcs_log("sndr", NULL,
 224                     gettext("%s: thread wait_sync_event started"), program);
 225 #endif
 226                 ;
 227         }
 228 
 229         for (;;) {
 230                 if (!first) {
 231                         first++;
 232                         (void) sleep(15);
 233                 } else
 234                         (void) sleep(MAIN_SLEEP);
 235 
 236                 bzero(rdc_info, size);
 237                 rdc_info->nset = rdc_maxsets;
 238                 if (RDC_IOCTL(RDC_STATUS, rdc_info, 0, 0, 0, 0, ustatus)
 239                     != SPCS_S_OK) {
 240                         spcs_log("sndr", &ustatus,
 241                             gettext("%s: status ioctl"),
 242                             program);
 243                         rdc_warn(&ustatus, gettext("status ioctl"));
 244                         continue;
 245                 }
 246 
 247                 cleanup_hostlist(rdc_info); /* remove non-existent hosts */
 248 
 249                 /*
 250                  * Check all enabled sets to see if a new remote host has
 251                  * appeared.
 252                  */
 253                 for (i = 0; i < rdc_maxsets; i++) {
 254                         if (!(rdc_info->rdc_set[i].flags & RDC_ENABLED))
 255                                 continue;
 256                         /* spawn a new thread for each new host found */
 257                         if (isnewhost(rdc_info->rdc_set[i].secondary.intf)) {
 258                                 /*
 259                                  * right now, we could be here before
 260                                  * the database did the write for this set
 261                                  * I could check the lock on the database
 262                                  * but I am just going to give up some time here
 263                                  * instead. Why do the allocations etc, etc
 264                                  * if the set is enabled in the kernel and not
 265                                  * in the config, we know that this set has the
 266                                  * lock. Why bother adding more contention to
 267                                  * the lock.
 268                                  * this is a daemon, afterall. its got time
 269                                  */
 270                                 (void) sleep(CFG_WAIT_SLEEP);
 271 
 272                                 spcs_log("sndr", NULL,
 273                                     gettext("%s: new host found (%s) starting "
 274                                     "its autosync thread"), program,
 275                                     rdc_info->rdc_set[i].secondary.intf);
 276 
 277                                 trc = thr_create(NULL, 0, wait_link_down,
 278                                     (void *) rdc_info->rdc_set[i].\
 279 secondary.intf, THR_BOUND|THR_DETACHED, NULL);
 280 
 281                                 if (trc != 0) {
 282                                         spcs_log("sndr", NULL,
 283                                             gettext(
 284                                             "%s create new autosync "
 285                                             "thread failed"), program);
 286                                         rdc_warn(NULL, gettext(
 287                                             "%s create new autosync "
 288                                             "thread failed"), program);
 289                                 }
 290                         }
 291                 }
 292         }
 293         /* NOTREACHED */
 294 }
 295 
 296 
 297 /*
 298  * The kernel wakes up this function every time it detects the link to the
 299  * specified host has dropped.
 300  */
 301 void *
 302 wait_link_down(void *thehost)
 303 {
 304         char *host = (char *)thehost;
 305         char tmphost[MAX_RDC_HOST_SIZE] = { '\0' };
 306         spcs_s_info_t ustatus;
 307 
 308         if (host)
 309                 (void) strncpy(tmphost, host, MAX_RDC_HOST_SIZE);
 310 
 311         ustatus = spcs_s_ucreate();
 312 
 313         /* Never give up */
 314         for (;;) {
 315 #ifdef DEBUG
 316                 spcs_log("sndr", NULL,
 317                     gettext("%s: awaiting link down ioctl for %s"),
 318                     program, host[0] == '\0' ? tmphost : host);
 319 #endif
 320                 if (RDC_IOCTL(RDC_LINK_DOWN, host, 0, 0, 0, 0, ustatus)
 321                     != SPCS_S_OK) {
 322                         spcs_log("sndr", &ustatus,
 323                             gettext("%s: link down ioctl"),
 324                             program);
 325                         rdc_warn(&ustatus, gettext("link down ioctl"));
 326                         continue;
 327                 }
 328 #ifdef DEBUG
 329 
 330                 spcs_log("sndr", NULL,
 331                     gettext("%s: received link down ioctl for %s"),
 332                     program, host[0] == '\0' ? tmphost : host);
 333 #endif
 334                 rdc_sync(host[0] == '\0' ? tmphost : host);
 335         }
 336         /* LINTED */
 337 }
 338 
 339 
 340 /*
 341  * Called when the link to the specified host has dropped.
 342  * For all Remote Mirror sets using the link that have autosync on,
 343  * issue rdcadm -u commands until they complete successfully.
 344  */
 345 void
 346 rdc_sync(char *tohost)
 347 {
 348         rdc_set_t *rdc_set = NULL;
 349         int *sync_done = NULL;
 350         int sets = 0;
 351         int syncs_done = 0;
 352         char cmd[256];
 353         rdc_config_t parms = { 0 };
 354         spcs_s_info_t ustatus;
 355         int i;
 356         int setnumber;
 357         int numfound = 0;
 358         char buf[CFG_MAX_BUF];
 359         char key[CFG_MAX_KEY];
 360         CFGFILE *cfg = NULL;
 361         int size;
 362         int first = 0;
 363         int death = 0;
 364         int cfglocked = 0;
 365 
 366         ustatus = spcs_s_ucreate();
 367 
 368         size = sizeof (rdc_set_t) * rdc_maxsets;
 369         rdc_set = malloc(size);
 370         if (rdc_set == NULL) {
 371                 spcs_log("sndr", NULL,
 372                     gettext("%s: unable to allocate %ld bytes"),
 373                     program, size);
 374                 rdc_warn(NULL,
 375                         gettext("unable to allocate %ld bytes"), size);
 376                 goto done;
 377         }
 378         bzero(rdc_set, size);
 379         size = sizeof (int) * rdc_maxsets;
 380         sync_done = malloc(size);
 381         if (sync_done == NULL) {
 382                 spcs_log("sndr", NULL,
 383                     gettext("%s: unable to allocate %ld bytes"),
 384                     program, size);
 385                 rdc_warn(NULL,
 386                         gettext("unable to allocate %ld bytes"), size);
 387                 goto done;
 388         }
 389         bzero(sync_done, size);
 390 
 391         /*
 392          * Get all sndr entries with shost matching tohost, and save the
 393          * details in an array.
 394          */
 395         for (i = 0; i < rdc_maxsets; i++) {
 396                 setnumber = i + 1;
 397                 bzero(buf, sizeof (buf));
 398                 bzero(key, sizeof (key));
 399 
 400                 (void) snprintf(key, sizeof (key), "sndr.set%d.shost",
 401                     setnumber);
 402 
 403                 if (!cfglocked) {
 404                         LOCKCFG();
 405                         if ((cfg = cfg_open(NULL)) == NULL) {
 406                                 spcs_log("sndr", NULL,
 407                                     gettext("%s: error opening config"),
 408                                     program);
 409 
 410                                 rdc_warn(NULL,
 411                                     gettext("error opening config"));
 412                                 UNLOCKCFG();
 413                                 goto done;
 414                         }
 415 
 416                         if (!cfg_lock(cfg, CFG_RDLOCK)) {
 417                                 spcs_log("sndr", NULL,
 418                                     gettext("%s: error locking config"),
 419                                     program);
 420                                 rdc_warn(NULL, gettext("error locking config"));
 421                                 goto done;
 422                         }
 423                 }
 424 
 425                 cfglocked = 1;
 426 
 427                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
 428                         if (numfound == 0) /* no matching hosts */
 429                                 death = 1; /* thread will exit */
 430                         break;
 431                 }
 432                 if (strcmp(buf, tohost) != 0)
 433                         continue;
 434 
 435                 numfound++;
 436                 (void) strncpy(rdc_set[sets].secondary.intf, buf,
 437                     MAX_RDC_HOST_SIZE);
 438 
 439                 /* Got a matching entry */
 440 
 441                 (void) snprintf(key, sizeof (key), "sndr.set%d.phost",
 442                     setnumber);
 443                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 444                         break;
 445                 (void) strncpy(rdc_set[sets].primary.intf, buf,
 446                     MAX_RDC_HOST_SIZE);
 447 
 448                 (void) snprintf(key, sizeof (key), "sndr.set%d.primary",
 449                     setnumber);
 450                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 451                         break;
 452                 (void) strncpy(rdc_set[sets].primary.file, buf, NSC_MAXPATH);
 453 
 454                 (void) snprintf(key, sizeof (key), "sndr.set%d.secondary",
 455                     setnumber);
 456                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 457                         break;
 458                 (void) strncpy(rdc_set[sets].secondary.file, buf, NSC_MAXPATH);
 459 
 460                 parms.command = RDC_CMD_STATUS;
 461                 bcopy((void *)(&rdc_set[sets]), (void *)(&parms.rdc_set[0]),
 462                     sizeof (rdc_set_t));
 463 
 464                 /*
 465                  * release cfg before diving into the kernel
 466                  * this prevents a possible deadlock when doing
 467                  * a reverse sync whick will wake up the sync_event
 468                  * thread which will try and iiadm -c and hang
 469                  * because we still have the cfg_lock. the timed
 470                  * wait cv in the kernel will fail the sync and things
 471                  * will undeadlock.
 472                  */
 473 
 474                 cfg_close(cfg);
 475                 cfg = NULL;
 476                 cfglocked = 0;
 477                 UNLOCKCFG();
 478 
 479                 if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
 480                         continue;
 481                 }
 482                 if ((parms.rdc_set[0].autosync == 0) ||
 483                     (!(parms.rdc_set[0].flags & RDC_LOGGING))) {
 484                         continue;
 485                 }
 486 
 487                 /* Found a suitable set with autosync on, in logging mode */
 488                 sets++;
 489         }
 490 
 491         if (cfg) {
 492                 cfg_close(cfg);
 493                 cfg = NULL;
 494                 UNLOCKCFG();
 495         }
 496 
 497         if (sets == 0) {
 498 #ifdef DEBUG
 499                 spcs_log("sndr", NULL,
 500                     gettext("%s: no sets requiring autosync found for %s"),
 501                     program, tohost);
 502 #endif
 503                 if (death) {
 504                         spcs_log("sndr", NULL,
 505                             gettext("%s: autosync thread stopping for %s "
 506                             "(host deconfigured)"), program, tohost);
 507                 }
 508                 goto done;
 509         }
 510 
 511         /* Keep issuing rdcadm -u commands until they have all completed */
 512         for (;;) {
 513                 if (!first)
 514                         first++;
 515                 else
 516                         (void) sleep(RESYNC_SLEEP);
 517 
 518                 /* Issue rdcadm -u commands for all remaining sets */
 519                 for (i = 0; i < sets; i++) {
 520                         if (sync_done[i])
 521                                 continue;
 522 
 523                         /*
 524                          * Need to check if autosync was turned off for a set
 525                          * while we were sleeping. We could have the case where
 526                          * an update sync failed and autosync was disabled
 527                          * while we were sleeping and didn't detect the disable.
 528                          * See BugID 4814213.
 529                          */
 530                         parms.command = RDC_CMD_STATUS;
 531                         bcopy((void *)(&rdc_set[i]),
 532                             (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
 533                         if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
 534                             ustatus) < 0) {
 535                                 spcs_log("sndr", &ustatus, gettext("%s: "
 536                                     "status not available for %s:%s, stopping "
 537                                     "this autosync attempt"), program, tohost,
 538                                     rdc_set[i].secondary.file);
 539                                 sync_done[i] = 1;
 540                                 syncs_done++;
 541                                 continue;
 542                         }
 543                         if (!(parms.rdc_set[0].autosync)) {
 544 #ifdef DEBUG
 545         spcs_log("sndr", NULL, gettext("%s: autosync disabled during sleep, "
 546             "stopping attempt for set %s:%s"), program, tohost,
 547             rdc_set[i].secondary.file);
 548 #endif
 549                                 sync_done[i] = 1;
 550                                 syncs_done++;
 551                                 continue;
 552                         }
 553 
 554                         (void) sprintf(cmd, "%s -un %s:%s", RDCADM, tohost,
 555                             rdc_set[i].secondary.file);
 556                         spcs_log("sndr", NULL,
 557                             gettext("%s: issuing update sync for %s:%s"),
 558                             program, tohost, rdc_set[i].secondary.file);
 559                         (void) system(cmd);
 560                 }
 561 
 562                 /* Issue rdcadm -w commands to wait for updates to finish */
 563                 for (i = 0; i < sets; i++) {
 564                         if (sync_done[i])
 565                                 continue;
 566 
 567                         (void) sprintf(cmd, "%s -wn %s:%s", RDCADM, tohost,
 568                             rdc_set[i].secondary.file);
 569                         spcs_log("sndr", NULL,
 570                             gettext("%s: issuing wait for %s:%s"),
 571                             program, tohost, rdc_set[i].secondary.file);
 572 
 573                         (void) system(cmd);
 574 
 575                         parms.command = RDC_CMD_STATUS;
 576                         bcopy((void *)(&rdc_set[i]),
 577                             (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
 578 
 579                         if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
 580                             ustatus) < 0) {
 581                                 spcs_log("sndr", &ustatus,
 582                                     gettext("%s: status not available for "
 583                                     "%s:%s, stopping this autosync attempt"),
 584                                     program, tohost, rdc_set[i].secondary.file);
 585                                 sync_done[i] = 1;
 586                                 syncs_done++;
 587                                 continue;
 588                         }
 589                         /* Check if completed OK, failed or autosync off */
 590                         if (!(parms.rdc_set[0].autosync) ||
 591                             !(parms.rdc_set[0].flags & RDC_LOGGING) &&
 592                             !(parms.rdc_set[0].flags & RDC_SYNCING)) {
 593                                 sync_done[i] = 1;
 594                                 syncs_done++;
 595                         }
 596                 }
 597 
 598                 if (syncs_done == sets)
 599                         break;          /* All completed OK */
 600         }
 601 
 602 done:
 603         if (cfg) {
 604                 cfg_close(cfg);
 605                 UNLOCKCFG();
 606         }
 607         spcs_s_ufree(&ustatus);
 608         if (sync_done)
 609                 free(sync_done);
 610         if (rdc_set)
 611                 free(rdc_set);
 612         if (death) { /* bye bye */
 613                 /*
 614                  * if perhaps we lost some race, lets remove this entry from
 615                  * the list. Then, if something did go wrong, and we did kill
 616                  * a valid thread, it will be detected on the next go around
 617                  * of the thread who is looking for new hosts to spawn threads
 618                  */
 619 
 620                 remove_from_hostlist(tohost);
 621                 thr_exit(0);
 622         }
 623 
 624         (void) sleep(RESYNC_SLEEP);
 625 }
 626 
 627 /*
 628  * Wait for notification by the kernel of a sync start or a sync completed OK
 629  */
 630 void *
 631 wait_sync_event()
 632 {
 633         spcs_s_info_t ustatus;
 634         char master[NSC_MAXPATH];
 635         char group[NSC_MAXPATH];
 636         int state;
 637 
 638         ustatus = spcs_s_ucreate();
 639 
 640         master[0] = '\0';
 641         group[0] = '\0';
 642 
 643         /* Never give up */
 644         for (;;) {
 645                 /* Kernel tells us which volume and group the event is for */
 646                 state = RDC_IOCTL(RDC_SYNC_EVENT, master, group, 0, 0, 0,
 647                     ustatus);
 648                 if (state < SPCS_S_OK) {
 649                         if (errno != EAGAIN) {
 650                                 spcs_log("sndr", &ustatus,
 651                                     gettext("%s: update ioctl"),
 652                                     program);
 653                                 rdc_warn(&ustatus, gettext("update ioctl"));
 654                                 continue;
 655                         }
 656                         master[0] = '\0';
 657                         continue;
 658                 }
 659 
 660                 /*
 661                  * If target is mounted at the start of a sync or reverse sync,
 662                  * return a negative ack.
 663                  */
 664                 if ((state == RDC_SYNC_START || state == RDC_RSYNC_START) &&
 665                     mounted(master)) {
 666                         spcs_log("sndr", NULL,
 667                             gettext("%s: %s has a file system mounted"),
 668                             program, master);
 669                         rdc_warn(NULL,
 670                             gettext("%s has a file system mounted"),
 671                             master);
 672                         master[0] = '\0';       /* negative ack */
 673                         continue;
 674                 }
 675 
 676                 switch (state) {
 677                 case RDC_SYNC_START:
 678                         if (group[0])
 679                                 group_start(group);
 680                         else
 681                                 sync_start(master);
 682                         break;
 683 
 684                 case RDC_SYNC_DONE:
 685                         if (group[0])
 686                                 group_complete(group);
 687                         else
 688                                 sync_complete(master);
 689                         break;
 690 
 691                 default:
 692                         break;
 693                 }
 694         }
 695         /* LINTED */
 696 }
 697 
 698 
 699 /*
 700  * A sync has completed OK to a volume not belonging to a group.
 701  * Set the state of the ndr_ii config entry to "update".
 702  */
 703 void
 704 sync_complete(char *master)
 705 {
 706         CFGFILE *cfg = NULL;
 707         char buf[CFG_MAX_BUF];
 708         char key[CFG_MAX_KEY];
 709         int i;
 710         int setnumber;
 711         int sev;
 712 
 713         LOCKCFG();
 714         if ((cfg = cfg_open(NULL)) == NULL) {
 715                 spcs_log("sndr", NULL,
 716                     gettext("%s: error opening config"),
 717                     program);
 718                 rdc_warn(NULL, gettext("error opening config"));
 719                 UNLOCKCFG();
 720                 return;
 721         }
 722         if (!cfg_lock(cfg, CFG_WRLOCK)) {
 723                 spcs_log("sndr", NULL,
 724                     gettext("%s: error locking config"),
 725                     program);
 726                 rdc_warn(NULL, gettext("error locking config"));
 727                 cfg_close(cfg);
 728                 UNLOCKCFG();
 729                 return;
 730         }
 731 
 732         /* get ndr_ii entries until a match is found */
 733         for (i = 0; ; i++) {
 734                 setnumber = i + 1;
 735 
 736                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
 737                     setnumber);
 738                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 739                         break;
 740                 if (strcmp(buf, master) != 0)
 741                         continue;
 742 
 743                 /* Found the matching entry */
 744 
 745                 /*
 746                  * Set state to "update" so that starting another sync will
 747                  * cause a new Point-in-Time Copy snapshot to be taken.
 748                  */
 749                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
 750                     setnumber);
 751                 if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
 752                     (cfg_commit(cfg) < 0)) {
 753                         spcs_log("sndr", NULL,
 754                             gettext("%s: unable to update \"%s\" "
 755                             "in configuration storage: %s"),
 756                             program, buf, cfg_error(&sev));
 757                         rdc_warn(NULL,
 758                             gettext("unable to update \"%s\" "
 759                             "in configuration storage: %s"),
 760                             buf, cfg_error(&sev));
 761                 }
 762                 break;
 763         }
 764 
 765         cfg_close(cfg);
 766         UNLOCKCFG();
 767 }
 768 
 769 
 770 /*
 771  * Starting a sync to the specified master volume.
 772  * Check the ndr_ii config entries to see if a Point-in-Time Copy
 773  * snapshot should be taken.
 774  */
 775 void
 776 sync_start(char *master)
 777 {
 778         char cmd[256];
 779         char buf[CFG_MAX_BUF];
 780         char key[CFG_MAX_KEY];
 781         CFGFILE *cfg = NULL;
 782         int i;
 783         int setnumber;
 784         int found;
 785         int sev;
 786         char shadow[NSC_MAXPATH];
 787         char bitmap[NSC_MAXPATH];
 788         char *ctag = NULL;
 789 
 790         LOCKCFG();
 791         if ((cfg = cfg_open(NULL)) == NULL) {
 792                 spcs_log("sndr", NULL,
 793                     gettext("%s: error opening config"),
 794                     program);
 795                 rdc_warn(NULL,
 796                     gettext("error opening config"));
 797                 UNLOCKCFG();
 798                 return;
 799         }
 800         if (!cfg_lock(cfg, CFG_RDLOCK)) {
 801                 spcs_log("sndr", NULL,
 802                     gettext("%s: error locking config"),
 803                     program);
 804                 rdc_warn(NULL, gettext("error locking config"));
 805                 cfg_close(cfg);
 806                 UNLOCKCFG();
 807                 return;
 808         }
 809 
 810         found = 0;
 811         /* get ndr_ii entries until a match is found */
 812         for (i = 0; ; i++) {
 813                 setnumber = i + 1;
 814 
 815                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
 816                     setnumber);
 817                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 818                         break;
 819                 if (strcmp(buf, master) != 0)
 820                         continue;
 821 
 822                 /* Got a matching entry */
 823 
 824                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
 825                     setnumber);
 826                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 827                         break;
 828                 (void) strncpy(shadow, buf, NSC_MAXPATH);
 829 
 830                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
 831                     setnumber);
 832                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 833                         break;
 834                 (void) strncpy(bitmap, buf, NSC_MAXPATH);
 835 
 836                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
 837                     setnumber);
 838                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 839                         break;
 840 
 841                 /*
 842                  * If an PIT snapshot has already been taken, and syncing did
 843                  * not complete, the state will be "noupdate", to indicate we
 844                  * should not take another one at this point.
 845                  */
 846                 if (strcmp(buf, NOUPDATE) != 0)
 847                         found = 1;
 848 
 849                 break;
 850         }
 851 
 852         if (!found) {
 853                 cfg_close(cfg);
 854                 UNLOCKCFG();
 855                 return;
 856         }
 857 
 858         found = 0;
 859         /* get ii entries until a match is found */
 860         for (i = 0; ; i++) {
 861                 setnumber = i + 1;
 862 
 863                 (void) snprintf(key, sizeof (key), "ii.set%d.shadow",
 864                     setnumber);
 865                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 866                         break;
 867                 if (strcmp(buf, shadow) != 0)
 868                         continue;
 869 
 870                 /* Matching shadow found, so ii already enabled */
 871                 found = 1;
 872                 break;
 873         }
 874 
 875         if (found) {
 876                 /* Already PIT enabled, so just take a snapshot */
 877 
 878                 /* Get cluster tag of matching entry */
 879                 (void) snprintf(key, sizeof (key), "ii.set%d.cnode", setnumber);
 880                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
 881                         if ((strlen(buf) == 0) || (buf[0] == '-'))
 882                                         ctag = "-C local";
 883                                 else
 884                                         ctag = "";
 885                 (void) sprintf(cmd, "%s %s -u s %s", IIADM, ctag, shadow);
 886         } else {
 887                 /*
 888                  * If clustered, need to enable PIT Copy sets in the same
 889                  * cluster as the Remote Mirror set
 890                  */
 891 
 892                 if (clustered) {
 893                         /* Find a RM set with master as the local volume */
 894 
 895                         for (i = 0; i < rdc_maxsets; i++) {
 896                                 setnumber = i + 1;
 897                                 (void) snprintf(key, sizeof (key),
 898                                     "sndr.set%d.phost", setnumber);
 899                                 if (cfg_get_cstring(cfg, key, buf,
 900                                     CFG_MAX_BUF) < 0)
 901                                         break;
 902 
 903                                 if (self_check(buf))
 904                                         (void) snprintf(key, sizeof (key),
 905                                             "sndr.set%d.primary", setnumber);
 906                                 else
 907                                         (void) snprintf(key, sizeof (key),
 908                                             "sndr.set%d.secondary", setnumber);
 909                                 if (cfg_get_cstring(cfg, key, buf,
 910                                     CFG_MAX_BUF) < 0)
 911                                         break;
 912 
 913                                 if (strcmp(buf, master) != 0)
 914                                         continue;
 915 
 916                                 /* Get cluster tag of matching entry */
 917 
 918                                 (void) snprintf(key, sizeof (key),
 919                                     "sndr.set%d.cnode", setnumber);
 920                                 if (cfg_get_cstring(cfg, key, buf,
 921                                     CFG_MAX_BUF) < 0)
 922                                         break;
 923                                 if ((strlen(buf) == 0) || (buf[0] == '-'))
 924                                         ctag = strdup("local");
 925                                 else
 926                                         ctag = strdup(buf);
 927                                 break;
 928                         }
 929                 }
 930 
 931                 /* Not already enabled, so enable a dependent */
 932                 if (ctag) {
 933                         (void) sprintf(cmd, "%s -C %s -e dep %s %s %s", IIADM,
 934                             ctag, master, shadow, bitmap);
 935                         free(ctag);
 936                 } else
 937                         (void) sprintf(cmd, "%s -e dep %s %s %s", IIADM, master,
 938                             shadow, bitmap);
 939         }
 940 
 941         cfg_close(cfg);
 942 
 943         if (system(cmd) != 0) {
 944                 spcs_log("sndr", NULL,
 945                     gettext("Point-in-Time Copy snapshot failed for %s %s %s."
 946                     " Please check validity of ndr_ii entry"),
 947                     master, shadow, bitmap);
 948                 cfg_close(cfg);
 949                 UNLOCKCFG();
 950                 return;
 951         }
 952 
 953         /*
 954          * PIT Copy enable or update was fine, so update the ndr_ii entry
 955          * to "noupdate", to prevent invalid point in time copies.
 956          */
 957 
 958         if ((cfg = cfg_open(NULL)) == NULL) {
 959                 spcs_log("sndr", NULL,
 960                     gettext("%s: error opening config"),
 961                     program);
 962                 rdc_warn(NULL,
 963                     gettext("error opening config"));
 964                 UNLOCKCFG();
 965                 return;
 966         }
 967         if (!cfg_lock(cfg, CFG_WRLOCK)) {
 968                 spcs_log("sndr", NULL,
 969                     gettext("%s: error locking config"),
 970                     program);
 971                 rdc_warn(NULL, gettext("error locking config"));
 972                 cfg_close(cfg);
 973                 UNLOCKCFG();
 974                 return;
 975         }
 976 
 977         /* get ndr_ii entries until a match is found */
 978         for (i = 0; ; i++) {
 979                 setnumber = i + 1;
 980 
 981                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
 982                     setnumber);
 983                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
 984                         break;
 985                 if (strcmp(buf, shadow) != 0)
 986                         continue;
 987 
 988                 /* Found the matching entry */
 989 
 990                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
 991                     setnumber);
 992                 if ((cfg_put_cstring(cfg, key, NOUPDATE,
 993                         strlen(NOUPDATE)) < 0) || (cfg_commit(cfg) < 0)) {
 994                         spcs_log("sndr", NULL,
 995                             gettext("%s: unable to update \"%s\" "
 996                             "in configuration storage: %s"),
 997                             program, buf, cfg_error(&sev));
 998                         rdc_warn(NULL,
 999                             gettext("unable to update \"%s\" "
1000                             "in configuration storage: %s"),
1001                             buf, cfg_error(&sev));
1002                 }
1003                 break;
1004         }
1005         cfg_close(cfg);
1006         UNLOCKCFG();
1007 }
1008 
1009 void
1010 cleanup_hostlist(rdc_status_t *rdc_info)
1011 {
1012         int i, j, k;
1013         char *host, *exhost;
1014 
1015 
1016         (void) mutex_lock(&host_list->hosts_mutex);
1017         for (i = 0; i < host_list->numhosts; i++) {
1018                 int found = 0;
1019                 for (j = 0; (j < rdc_maxsets) && !found; j++) {
1020                         if (!rdc_info->rdc_set[j].flags & RDC_ENABLED)
1021                                 continue;
1022                         if ((!host_list->configured[i]) ||
1023                             (host_list->hosts[i] == NULL)) {
1024                                 (void) mutex_unlock(&host_list->hosts_mutex);
1025                                 return;
1026                         }
1027 
1028                         host = rdc_info->rdc_set[j].secondary.intf;
1029                         if (strcmp(host_list->hosts[i], host) == 0)
1030                                 found++;
1031                 }
1032                 if (j == rdc_maxsets) {
1033                         /*
1034                          * this set is not in the kernel, so remove from list
1035                          */
1036                         exhost = host_list->hosts[i];
1037                         if (exhost) {
1038                                 free(exhost);
1039                                 exhost = NULL;
1040                         }
1041 
1042                         k = i;
1043                         while (k < host_list->numhosts) {
1044                             host_list->hosts[k] = k < host_list->numhosts - 1 ?
1045                                 host_list->hosts[k+1] : NULL;
1046                             k++;
1047                         }
1048                         host_list->numhosts--;
1049 
1050                         bcopy(&host_list->configured[i+1],
1051                             &host_list->configured[i],
1052                             (MAXHOSTS - i + 1) * sizeof (int));
1053                         host_list->configured[MAXHOSTS - 1] = 0;
1054                 }
1055         }
1056         (void) mutex_unlock(&host_list->hosts_mutex);
1057 }
1058 
1059 /*
1060  * explicity remove a host from the host list
1061  * also update the configured array
1062  * called in rdc_sync, just before exiting a thread.
1063  */
1064 void
1065 remove_from_hostlist(char *host)
1066 {
1067         int i, k;
1068         char *exhost;
1069 
1070         /* why bother? */
1071         if ((!host) || (host[0] == '\0'))
1072                 return;
1073 
1074         (void) mutex_lock(&host_list->hosts_mutex);
1075         for (i = 0; i < host_list->numhosts; i++) {
1076                 if (strcmp(host, host_list->hosts[i]) == 0) { /* found it */
1077                         exhost = host_list->hosts[i];
1078                         if (exhost) {
1079                                 free(exhost);
1080                                 exhost = NULL;
1081                         }
1082                         k = i;
1083                         while (k < host_list->numhosts) {
1084                             host_list->hosts[k] = k < host_list->numhosts - 1 ?
1085                                 host_list->hosts[k+1] : NULL;
1086                             k++;
1087                         }
1088                         host_list->numhosts--;
1089                         bcopy(&host_list->configured[i+1],
1090                             &host_list->configured[i],
1091                             (MAXHOSTS - i + 1) * sizeof (int));
1092                         host_list->configured[MAXHOSTS - 1] = 0;
1093                 }
1094 
1095         }
1096         (void) mutex_unlock(&host_list->hosts_mutex);
1097 }
1098 /*
1099  * Check to see if this host isn't in our list, so needs a new rdcsyncd proc
1100  */
1101 int
1102 isnewhost(char *host)
1103 {
1104         int i;
1105         int new;
1106 
1107         if (self_check(host)) {
1108                 return (0);
1109         }
1110 
1111         (void) mutex_lock(&host_list->hosts_mutex);
1112         new = 1;
1113         for (i = 0; i < MAXHOSTS; i++) {
1114                 if (host_list->configured[i] == 0) {
1115                         host_list->configured[i] = 1;
1116                         host_list->hosts[i] = strdup(host);
1117                         host_list->numhosts++;
1118                         break;
1119                 }
1120                 if (strcmp(host, host_list->hosts[i]) == 0) {
1121                         new = 0;
1122                         break;
1123                 }
1124         }
1125         (void) mutex_unlock(&host_list->hosts_mutex);
1126         if (i == MAXHOSTS)
1127                 new = 0;
1128         return (new);
1129 }
1130 
1131 
1132 /*
1133  * Look for a matching volume name in our remembered list.
1134  */
1135 int
1136 volume_match(char *buf, char **volume_list, int volumes)
1137 {
1138         int i;
1139         char *vol;
1140 
1141         for (i = 0; i < volumes; i++) {
1142                 vol = volume_list[i];
1143                 if (strcmp(buf, vol) == 0) {
1144                         return (1);
1145                 }
1146         }
1147         return (0);
1148 }
1149 
1150 
1151 /*
1152  * A sync has completed to a group. We can only update the ndr_ii entries
1153  * if all the members of the group have completed their syncs OK.
1154  * It would be bad to allow some members of the group to have PIT Copy snapshots
1155  * taken and others not, as they need to be consistent.
1156  */
1157 void
1158 group_complete(char *group)
1159 {
1160         char **volumes = NULL;
1161         spcs_s_info_t ustatus;
1162         rdc_config_t parms = { 0 };
1163         char buf[CFG_MAX_BUF];
1164         char key[CFG_MAX_KEY];
1165         CFGFILE *cfg = NULL;
1166         int i;
1167         int setnumber;
1168         int found;
1169         int replicating = 0;
1170         char primary[NSC_MAXPATH];
1171         char secondary[NSC_MAXPATH];
1172         char phost[MAX_RDC_HOST_SIZE];
1173         char shost[MAX_RDC_HOST_SIZE];
1174         rdc_set_t *rdc_set;
1175         int sev;
1176         char *local_file;
1177         int size;
1178 
1179         ustatus = spcs_s_ucreate();
1180 
1181         size = sizeof (char *) * rdc_maxsets;
1182         volumes = malloc(size);
1183         if (volumes == NULL) {
1184                 spcs_log("sndr", NULL,
1185                     gettext("%s: unable to allocate %ld bytes"),
1186                     program, size);
1187                 rdc_warn(NULL,
1188                         gettext("unable to allocate %ld bytes"), size);
1189                 goto done;
1190         }
1191         bzero(volumes, size);
1192 
1193         /*
1194          * If all members of this group are replicating
1195          * set ii_ndr state to "update". Otherwise leave them alone.
1196          */
1197         LOCKCFG();
1198         if ((cfg = cfg_open(NULL)) == NULL) {
1199                 spcs_log("sndr", NULL,
1200                     gettext("%s: error opening lconfig"),
1201                     program);
1202                 rdc_warn(NULL, gettext("error opening config"));
1203                 UNLOCKCFG();
1204                 goto done;
1205         }
1206 
1207         if (!cfg_lock(cfg, CFG_RDLOCK)) {
1208                 spcs_log("sndr", NULL,
1209                     gettext("%s: error locking config"),
1210                     program);
1211                 rdc_warn(NULL, gettext("error locking config"));
1212                 goto done;
1213         }
1214 
1215         found = 0;
1216 
1217         /* get all RM entries, with a matching group, that are replicating */
1218         for (i = 0; i < rdc_maxsets; i++) {
1219                 setnumber = i + 1;
1220 
1221                 (void) snprintf(key, sizeof (key),
1222                     "sndr.set%d.group", setnumber);
1223                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1224                         break;
1225 
1226                 if (strcmp(buf, group) != 0)
1227                         continue;
1228 
1229                 /* Found a matching entry */
1230 
1231                 (void) snprintf(key, sizeof (key),
1232                     "sndr.set%d.primary", setnumber);
1233                 if (cfg_get_cstring(cfg, key, primary, sizeof (primary)) < 0)
1234                         break;
1235                 (void) strcpy(parms.rdc_set->primary.file, primary);
1236 
1237                 (void) snprintf(key, sizeof (key),
1238                     "sndr.set%d.phost", setnumber);
1239                 if (cfg_get_cstring(cfg, key, phost, sizeof (phost)) < 0)
1240                         break;
1241                 (void) strcpy(parms.rdc_set->primary.intf, phost);
1242 
1243                 (void) snprintf(key, sizeof (key),
1244                     "sndr.set%d.secondary", setnumber);
1245                 if (cfg_get_cstring(cfg, key, secondary,
1246                                 sizeof (secondary)) < 0)
1247                         break;
1248                 (void) strcpy(parms.rdc_set->secondary.file, secondary);
1249 
1250                 (void) snprintf(key, sizeof (key),
1251                     "sndr.set%d.shost", setnumber);
1252                 if (cfg_get_cstring(cfg, key, shost, sizeof (shost)) < 0)
1253                         break;
1254                 (void) strcpy(parms.rdc_set->secondary.intf, shost);
1255 
1256                 parms.command = RDC_CMD_STATUS;
1257                 if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
1258                         continue;
1259                 }
1260 
1261                 /* We found a matching set */
1262                 found++;
1263 
1264                 if (self_check(phost))
1265                         local_file = primary;
1266                 else
1267                         local_file = secondary;
1268 
1269                 rdc_set = &parms.rdc_set[0];
1270                 if (!(rdc_set->flags & RDC_LOGGING) &&
1271                     !(rdc_set->flags & RDC_SYNCING)) {
1272                         volumes[replicating] = strdup(local_file);
1273                         if (volumes[replicating] == NULL) {
1274                                 size = strlen(local_file);
1275                                 spcs_log("sndr", NULL,
1276                                     gettext("%s: unable to allocate %ld bytes"),
1277                                     program, size);
1278                                 rdc_warn(NULL,
1279                                     gettext("unable to allocate %ld bytes"),
1280                                     size);
1281                                 goto done;
1282                         }
1283                         /* We remember all replicating sets */
1284                         replicating++;
1285                 } else
1286                         break;          /* Not all replicating, so done */
1287         }
1288 
1289         if (found != replicating)
1290                 goto done;
1291 
1292         /* All replicating, so update ndr_ii state fields */
1293 
1294         cfg_unlock(cfg);
1295 
1296         if (!cfg_lock(cfg, CFG_WRLOCK)) {
1297                 spcs_log("sndr", NULL,
1298                     gettext("%s: error locking lconfig"),
1299                     program);
1300                 rdc_warn(NULL, gettext("error locking config"));
1301                 goto done;
1302         }
1303 
1304         /*
1305          * Search through the ndr_ii entries for entries
1306          * that match the saved secondary volume names.
1307          * Set state to "update".
1308          */
1309 
1310         for (i = 0; ; i++) {
1311                 setnumber = i + 1;
1312 
1313                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1314                     setnumber);
1315                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1316                         break;
1317 
1318                 if (!volume_match(buf, volumes, found)) {
1319                         continue;
1320                 }
1321 
1322                 /* Got a matching entry */
1323 
1324                 (void) snprintf(key, sizeof (key),
1325                     "ndr_ii.set%d.state", setnumber);
1326                 if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
1327                     (cfg_commit(cfg) < 0)) {
1328                         spcs_log("sndr", NULL,
1329                             gettext("%s: unable to update \"%s\" "
1330                             "in configuration storage: %s"),
1331                             program, buf, cfg_error(&sev));
1332                         rdc_warn(NULL,
1333                             gettext("unable to update \"%s\" "
1334                             "in configuration storage: %s"),
1335                             buf, cfg_error(&sev));
1336                 }
1337         }
1338 
1339 
1340 done:
1341         if (cfg) {
1342                 cfg_close(cfg);
1343                 UNLOCKCFG();
1344         }
1345         spcs_s_ufree(&ustatus);
1346         if (volumes) {
1347                 for (i = 0; i < replicating; i++)
1348                         free(volumes[i]);
1349                 free(volumes);
1350         }
1351 }
1352 
1353 
1354 /*
1355  * Sync started to a member of a group.
1356  * If all members of the group are in ndr_ii state "update" then take an PIT
1357  * snapshot on all of them. This will provide a consistent point-in-time
1358  * copy until whatever syncs take place are all completed.
1359  */
1360 void
1361 group_start(char *group)
1362 {
1363         char **masters = NULL;
1364         char **shadows = NULL;
1365         char **bitmaps = NULL;
1366         char cmd[256];
1367         char buf[CFG_MAX_BUF];
1368         char key[CFG_MAX_KEY];
1369         CFGFILE *cfg = NULL;
1370         int i;
1371         int j;
1372         int setnumber;
1373         int found;
1374         int sndr_sets = 0;
1375         int update_needed = 0;
1376         int sev;
1377         char *ctag = NULL;
1378         int commit = 0;
1379         int size;
1380 
1381         size = sizeof (char *) * rdc_maxsets;
1382         masters = malloc(size);
1383         if (masters == NULL) {
1384                 spcs_log("sndr", NULL,
1385                     gettext("%s: unable to allocate %ld bytes"),
1386                     program, size);
1387                 rdc_warn(NULL,
1388                         gettext("unable to allocate %ld bytes"), size);
1389                 goto done;
1390         }
1391         bzero(masters, size);
1392         shadows = malloc(size);
1393         if (shadows == NULL) {
1394                 spcs_log("sndr", NULL,
1395                     gettext("%s: unable to allocate %ld bytes"),
1396                     program, size);
1397                 rdc_warn(NULL,
1398                         gettext("unable to allocate %ld bytes"), size);
1399                 goto done;
1400         }
1401         bzero(shadows, size);
1402         bitmaps = malloc(size);
1403         if (bitmaps == NULL) {
1404                 spcs_log("sndr", NULL,
1405                     gettext("%s: unable to allocate %ld bytes"),
1406                     program, size);
1407                 rdc_warn(NULL,
1408                         gettext("unable to allocate %ld bytes"), size);
1409                 goto done;
1410         }
1411         bzero(bitmaps, size);
1412 
1413         LOCKCFG();
1414         if ((cfg = cfg_open(NULL)) == NULL) {
1415                 spcs_log("sndr", NULL,
1416                     gettext("%s: error opening config"),
1417                     program);
1418                 rdc_warn(NULL,
1419                     gettext("error opening config"));
1420                 UNLOCKCFG();
1421                 goto done;
1422         }
1423 
1424         if (!cfg_lock(cfg, CFG_WRLOCK)) {
1425                 spcs_log("sndr", NULL,
1426                     gettext("%s: error locking config"),
1427                     program);
1428                 rdc_warn(NULL, gettext("error locking config"));
1429                 goto done;
1430         }
1431 
1432         /* Now get all Remote Mirror entries with a matching group */
1433         for (i = 0; i < rdc_maxsets; i++) {
1434                 setnumber = i + 1;
1435 
1436                 (void) snprintf(key, sizeof (key),
1437                     "sndr.set%d.group", setnumber);
1438                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1439                         break;
1440 
1441                 if (strcmp(buf, group) != 0)
1442                         continue;
1443 
1444                 /* Found a matching entry */
1445 
1446                 (void) snprintf(key, sizeof (key),
1447                     "sndr.set%d.phost", setnumber);
1448                 if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1449                         break;
1450 
1451                 if (self_check(buf)) {
1452                         (void) snprintf(key, sizeof (key), "sndr.set%d.primary",
1453                             setnumber);
1454                 } else {
1455                         (void) snprintf(key, sizeof (key),
1456                             "sndr.set%d.secondary", setnumber);
1457                 }
1458                 if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1459                         break;
1460 
1461                 masters[sndr_sets] = strdup(buf);
1462                 if (masters[sndr_sets] == NULL) {
1463                         size = strlen(buf);
1464                         spcs_log("sndr", NULL,
1465                             gettext("%s: unable to allocate %ld bytes"),
1466                             program, size);
1467                         rdc_warn(NULL,
1468                                 gettext("unable to allocate %ld bytes"), size);
1469                         goto done;
1470                 }
1471                 sndr_sets++;
1472 
1473                 if (ctag == NULL && clustered) {
1474                         /* Get cluster tag of matching entry */
1475 
1476                         (void) snprintf(key, sizeof (key), "sndr.set%d.cnode",
1477                             setnumber);
1478                         if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
1479                                 ctag = strdup(buf);
1480                 }
1481         }
1482 
1483         /*
1484          * Search through the ndr_ii entries for entries
1485          * that match the saved local volume names and are in "update" state.
1486          */
1487 
1488         update_needed = 0;
1489 
1490         for (i = 0; ; i++) {
1491                 setnumber = i + 1;
1492 
1493                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1494                     setnumber);
1495                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1496                         break;
1497 
1498                 if (!volume_match(buf, masters, sndr_sets))
1499                         continue;
1500 
1501                 /* Got a matching entry */
1502 
1503                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
1504                     setnumber);
1505                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1506                         break;
1507                 shadows[update_needed] = strdup(buf);
1508                 if (shadows[update_needed] == NULL) {
1509                         size = strlen(buf);
1510                         spcs_log("sndr", NULL,
1511                             gettext("%s: unable to allocate %ld bytes"),
1512                             program, size);
1513                         rdc_warn(NULL,
1514                                 gettext("unable to allocate %ld bytes"), size);
1515                         goto done;
1516                 }
1517 
1518                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
1519                     setnumber);
1520                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1521                         break;
1522                 }
1523                 bitmaps[update_needed] = strdup(buf);
1524                 if (bitmaps[update_needed] == NULL) {
1525                         size = strlen(buf);
1526                         spcs_log("sndr", NULL,
1527                             gettext("%s: unable to allocate %ld bytes"),
1528                             program, size);
1529                         rdc_warn(NULL,
1530                                 gettext("unable to allocate %ld bytes"), size);
1531                         goto done;
1532                 }
1533 
1534                 (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1535                     setnumber);
1536                 if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1537                         break;
1538                 }
1539                 if (strcmp(buf, UPDATE) != 0) {
1540                         break;
1541                 }
1542 
1543                 update_needed++;
1544         }
1545 
1546         if (update_needed != sndr_sets) {
1547 #ifdef DEBUG
1548                 spcs_log("sndr", NULL,
1549                     gettext("%s: group sync: no Point-in-Time Copy snapshot "
1550                             "for %s"), program, group);
1551 #endif
1552                 goto done;
1553         }
1554 
1555         /* All RM sets in the group have an ndr_ii entry in "update" state */
1556 
1557         /* Issue PIT Copy snapshot commands for all sets in the group */
1558         for (j = 0; j < sndr_sets; j++) {
1559                 found = 0;
1560 
1561                 /* get ii entries until a match is found */
1562                 for (i = 0; ; i++) {
1563                         setnumber = i + 1;
1564 
1565                         (void) snprintf(key, sizeof (key), "ii.set%d.shadow",
1566                             setnumber);
1567                         if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1568                                 break;
1569                         if (strcmp(buf, shadows[j]) != 0)
1570                                 continue;
1571 
1572                         /* Matching shadow found, so ii already enabled */
1573                         found = 1;
1574                         break;
1575                 }
1576 
1577                 if (commit)
1578                         if (cfg_commit(cfg) < 0)
1579                                 rdc_warn(NULL, gettext("commit config error"));
1580                 cfg_close(cfg);
1581 
1582                 if (found) {
1583                         (void) sprintf(cmd, "%s -u s %s", IIADM, shadows[j]);
1584                 } else {
1585                         if (ctag) {
1586                                 (void) sprintf(cmd, "%s -C %s -e dep %s %s %s",
1587                                     IIADM, ctag, masters[j], shadows[j],
1588                                     bitmaps[j]);
1589                                 free(ctag);
1590                                 ctag = NULL;
1591                         } else
1592                                 (void) sprintf(cmd, "%s -e dep %s %s %s", IIADM,
1593                                     masters[j], shadows[j], bitmaps[j]);
1594                 }
1595 
1596                 if (system(cmd) != 0) {
1597                         spcs_log("sndr", NULL,
1598                             gettext("%s: group sync: Point-in-Time Copy"
1599                             " snapshot failed for %s"),
1600                             program, masters[j]);
1601 
1602                         goto done;
1603                 }
1604 
1605                 if ((cfg = cfg_open(NULL)) == NULL) {
1606                         spcs_log("sndr", NULL,
1607                             gettext("%s: error opening config"),
1608                             program);
1609                         rdc_warn(NULL,
1610                             gettext("error opening config"));
1611                         goto done;
1612                 }
1613                 if (!cfg_lock(cfg, CFG_WRLOCK)) {
1614                         spcs_log("sndr", NULL,
1615                             gettext("%s: error locking config"),
1616                             program);
1617                         rdc_warn(NULL, gettext("error locking config"));
1618                         goto done;
1619                 }
1620                 commit = 0;
1621 
1622                 /* PIT enable or update was fine, so update the ndr_ii entry */
1623 
1624                 /* get ndr_ii entries until a match is found */
1625                 for (i = 0; ; i++) {
1626                         setnumber = i + 1;
1627 
1628                         (void) snprintf(key, sizeof (key),
1629                             "ndr_ii.set%d.shadow", setnumber);
1630                         if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1631                                 break;
1632                         if (strcmp(buf, shadows[j]) != 0)
1633                                 continue;
1634 
1635                         /* Found the matching entry */
1636 
1637                         (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1638                             setnumber);
1639                         if (cfg_put_cstring(cfg, key, NOUPDATE,
1640                             strlen(NOUPDATE)) < 0) {
1641                                 spcs_log("sndr", NULL,
1642                                     gettext("%s: unable to update \"%s\" "
1643                                     "in configuration storage: %s"),
1644                                     program, buf, cfg_error(&sev));
1645                                 rdc_warn(NULL,
1646                                     gettext("unable to update \"%s\" "
1647                                     "in configuration storage: %s"),
1648                                     buf, cfg_error(&sev));
1649                         } else
1650                                 commit = 1;
1651                         break;
1652                 }
1653         }
1654 
1655         if (commit)
1656                 if (cfg_commit(cfg) < 0)
1657                         rdc_warn(NULL, gettext("commit config error"));
1658 
1659         spcs_log("sndr", NULL,
1660             gettext("%s: group sync: Point-in-Time Copy snapshots completed "
1661                     "for %s"), program, group);
1662 
1663 done:
1664         if (ctag)
1665                 free(ctag);
1666 
1667         if (cfg) {
1668                 cfg_close(cfg);
1669                 UNLOCKCFG();
1670         }
1671 
1672         if (masters) {
1673                 for (i = 0; i < sndr_sets; i++) {
1674                         if (masters[i])
1675                                 free(masters[i]);
1676                 }
1677                 free(masters);
1678         }
1679 
1680         if (shadows) {
1681                 for (i = 0; i < update_needed; i++) {
1682                         if (shadows[i])
1683                                 free(shadows[i]);
1684                 }
1685                 free(shadows);
1686         }
1687 
1688         if (bitmaps) {
1689                 for (i = 0; i < update_needed; i++) {
1690                         if (bitmaps[i])
1691                                 free(bitmaps[i]);
1692                 }
1693                 free(bitmaps);
1694         }
1695 }