1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/types.h>
  27 #include <sys/ksynch.h>
  28 #include <sys/cmn_err.h>
  29 #include <sys/errno.h>
  30 #include <sys/kmem.h>
  31 #include <sys/ddi.h>
  32 
  33 #include <sys/nsc_thread.h>
  34 #include "sd_bcache.h"
  35 #include "sd_ft.h"
  36 #include "sd_misc.h"
  37 #include "sd_pcu.h"
  38 #include "sd_io.h"
  39 #include "sd_bio.h"
  40 #include "sd_trace.h"
  41 #include "sd_tdaemon.h"
  42 #include <sys/nsctl/nsctl.h>
  43 
  44 #include <sys/unistat/spcs_s.h>
  45 #include <sys/unistat/spcs_s_k.h>
  46 #include <sys/unistat/spcs_errors.h>
  47 #include <sys/nsctl/safestore.h>
  48 
  49 extern int sdbc_use_dmchain;
  50 
  51 int _sd_cblock_shift = 0;
  52 
  53 int _SD_SELF_HOST = _SD_NO_HOST;
  54 int _SD_MIRROR_HOST = _SD_NO_HOST;
  55 int _SD_NUM_REM;
  56 int _sd_nodes_configured;
  57 int _sdbc_gateway_wblocks;
  58 
  59 int _SD_NETS = 0;
  60 
  61 /*
  62  * Normally we unregister memory at deconfig time. By setting this non-zero
  63  * it will be delayed until unload time.
  64  */
  65 int _sdbc_memtype_deconfigure_delayed = 0;
  66 
  67 nsc_mem_t *sdbc_iobuf_mem, *sdbc_hash_mem;
  68 nsc_mem_t *sdbc_local_mem, *sdbc_stats_mem, *sdbc_cache_mem;
  69 nsc_mem_t *sdbc_info_mem;
  70 
  71 _sd_cache_param_t _sd_cache_config;
  72 
  73 kmutex_t _sdbc_config_lock;
  74 volatile int _sd_cache_dem_cnt;
  75 
  76 #if !defined(m88k) || defined(lint)
  77 volatile int _sd_cache_initialized;
  78 #endif
  79 
  80 static blind_t sdbc_power;
  81 
  82 static
  83 nsc_def_t _sdbc_power_def[] = {
  84         "Power_Lost",   (uintptr_t)_sdbc_power_lost,    0,
  85         "Power_OK",     (uintptr_t)_sdbc_power_ok,      0,
  86         "Power_Down",   (uintptr_t)_sdbc_power_down,    0,
  87         0,              0,              0
  88 };
  89 
  90 /*
  91  * Forward declare all statics that are used before defined to enforce
  92  * parameter checking
  93  * Some (if not all) of these could be removed if the code were reordered
  94  */
  95 
  96 int _sd_fill_pattern(caddr_t addr, uint_t pat, uint_t size);
  97 static void _sdbc_nodeid_deconfigure(void);
  98 static void _sdbc_nodeid_configure(void);
  99 static void _sdbc_thread_deconfigure(void);
 100 static int _sdbc_thread_configure(void);
 101 void sst_deinit();
 102 
 103 ss_common_config_t safestore_config;
 104 safestore_ops_t *sdbc_safestore;
 105 
 106 /*
 107  * _sdbc_memtype_configure - register with the sd layer the types of memory
 108  * we want to use. If any of the critical memory types can't be registered
 109  * we return non-zero otherwise 0.
 110  */
 111 static int
 112 _sdbc_memtype_configure(void)
 113 {
 114 
 115         if ((sdbc_info_mem = nsc_register_mem("sdbc:info",
 116             NSC_MEM_GLOBAL, KM_NOSLEEP)) == NULL) {
 117                 return (EINVAL);
 118         }
 119 
 120         sdbc_local_mem = nsc_register_mem("sdbc:local", NSC_MEM_LOCAL, 0);
 121         sdbc_stats_mem = nsc_register_mem("sdbc:stats", NSC_MEM_LOCAL, 0);
 122         sdbc_iobuf_mem = nsc_register_mem("sdbc:iobuf", NSC_MEM_LOCAL, 0);
 123 
 124         sdbc_cache_mem = nsc_register_mem("sdbc:cache", NSC_MEM_LOCAL, 0);
 125 
 126         sdbc_hash_mem = nsc_register_mem("sdbc:hash", NSC_MEM_LOCAL, 0);
 127 
 128         return (0);
 129 }
 130 
 131 /*
 132  * _sdbc_memtype_deconfigure - undo the effects of _sdbc_memtype_configure.
 133  */
 134 void
 135 _sdbc_memtype_deconfigure(void)
 136 {
 137 
 138         if (sdbc_hash_mem)
 139                 nsc_unregister_mem(sdbc_hash_mem);
 140         if (sdbc_iobuf_mem)
 141                 nsc_unregister_mem(sdbc_iobuf_mem);
 142         if (sdbc_cache_mem)
 143                 nsc_unregister_mem(sdbc_cache_mem);
 144         if (sdbc_stats_mem)
 145                 nsc_unregister_mem(sdbc_stats_mem);
 146         if (sdbc_local_mem)
 147                 nsc_unregister_mem(sdbc_local_mem);
 148         if (sdbc_info_mem)
 149                 nsc_unregister_mem(sdbc_info_mem);
 150 
 151         sdbc_info_mem = NULL;
 152         sdbc_local_mem = sdbc_stats_mem = sdbc_cache_mem = NULL;
 153         sdbc_iobuf_mem = sdbc_hash_mem = NULL;
 154 
 155 }
 156 
 157 
 158 /*
 159  * figure out what kind of safe storage we need
 160  */
 161 uint_t
 162 sdbc_determine_safestore()
 163 {
 164         return (SS_M_RAM | SS_T_NONE);
 165 }
 166 
 167 static void
 168 sd_setup_ssconfig()
 169 {
 170         safestore_config.ssc_client_psize = BLK_SIZE(1);
 171 
 172         if (_sd_cache_config.write_cache)
 173                 safestore_config.ssc_wsize =
 174                     _sd_cache_config.write_cache * MEGABYTE;
 175         else
 176                 safestore_config.ssc_wsize =
 177                     (_sd_cache_config.cache_mem[_SD_NO_NET] * MEGABYTE)/2;
 178         safestore_config.ssc_maxfiles = sdbc_max_devs;
 179         safestore_config.ssc_pattern = _sd_cache_config.fill_pattern;
 180         safestore_config.ssc_flag = _sd_cache_config.gen_pattern ?
 181             SS_GENPATTERN : 0;
 182 }
 183 
 184 /*
 185  * _sdbc_configure - process the ioctl that describes the configuration
 186  * for the cache. This is the main driver routine for cache configuration
 187  * Return 0 on success, otherwise nonzero.
 188  *
 189  */
 190 int
 191 _sdbc_configure(_sd_cache_param_t *uptr,
 192         _sdbc_config_t *mgmt, spcs_s_info_t spcs_kstatus)
 193 {
 194         int cache_bytes;
 195         nsc_io_t *io;
 196         char itmp[16];
 197         char itmp2[16];
 198         int i;
 199         uint_t ss_type;
 200         int rc;
 201 
 202         ASSERT(MUTEX_HELD(&_sdbc_config_lock));
 203 
 204         _sd_print(1, "sdbc(_sdbc_configure) _SD_MAGIC 0x%x\n", _SD_MAGIC);
 205 
 206         _sd_ioset = 0;
 207         if (_sd_cache_initialized) {
 208                 spcs_s_add(spcs_kstatus, SDBC_EALREADY);
 209                 rc = EALREADY;
 210                 goto out;
 211         }
 212 
 213         ASSERT((uptr != NULL) || (mgmt != NULL));
 214 
 215         if (uptr) {
 216                 if (copyin(uptr, &_sd_cache_config,
 217                     sizeof (_sd_cache_param_t))) {
 218                         rc = EFAULT;
 219                         goto out;
 220                 }
 221         } else {
 222                 bzero(&_sd_cache_config, sizeof (_sd_cache_config));
 223 
 224                 /* copy in mgmt config info */
 225 
 226                 _sd_cache_config.magic = mgmt->magic;
 227                 _sd_cache_config.threads = mgmt->threads;
 228 
 229                 for (i = 0; i < CACHE_MEM_PAD; i++) {
 230                         _sd_cache_config.cache_mem[i] = mgmt->cache_mem[i];
 231                 }
 232 
 233                 /* fake the rest as a single node config */
 234 
 235                 _sd_cache_config.nodes_conf[0] = nsc_node_id();
 236                 _sd_cache_config.num_nodes = 1;
 237         }
 238 
 239         /*
 240          * Check that the requested cache size doesn't break the code.
 241          * This test can be refined once the cache size is stored in variables
 242          * larger than an int.
 243          */
 244         for (i = 0; i < MAX_CACHE_NET; i++) {
 245                 if (_sd_cache_config.cache_mem[i] < 0) {
 246                         cmn_err(CE_WARN, "!_sdbc_configure: "
 247                             "negative cache size (%d) for net %d",
 248                             _sd_cache_config.cache_mem[i], i);
 249                         spcs_s_add(spcs_kstatus, SDBC_ENONETMEM);
 250                         rc = SDBC_ENONETMEM;
 251                         goto out;
 252                 }
 253                 if (_sd_cache_config.cache_mem[i] > MAX_CACHE_SIZE) {
 254                         _sd_cache_config.cache_mem[i] = MAX_CACHE_SIZE;
 255                         cmn_err(CE_WARN, "!_sdbc_configure: "
 256                             "cache size limited to %d megabytes for net %d",
 257                             MAX_CACHE_SIZE, i);
 258                 }
 259         }
 260 
 261         if (_sd_cache_config.blk_size == 0)
 262                 _sd_cache_config.blk_size = 8192;
 263 
 264         if (_sd_cache_config.procs == 0)
 265                 _sd_cache_config.procs = 16;
 266 
 267 #if !defined(_SD_8K_BLKSIZE)
 268         if (_sd_cache_config.blk_size != 4096) {
 269 #else
 270         if (_sd_cache_config.blk_size != 8192) {
 271 #endif
 272                 (void) spcs_s_inttostring(_sd_cache_config.blk_size, itmp,
 273                     sizeof (itmp), 0);
 274                 spcs_s_add(spcs_kstatus, SDBC_ESIZE, itmp);
 275                 rc = SDBC_EENABLEFAIL;
 276                 goto out;
 277         }
 278         if (((_sd_cblock_shift =
 279             get_high_bit(_sd_cache_config.blk_size)) == -1) ||
 280             (_sd_cache_config.blk_size != (1 << _sd_cblock_shift))) {
 281                 (void) spcs_s_inttostring(_sd_cache_config.blk_size, itmp,
 282                     sizeof (itmp), 0);
 283                 spcs_s_add(spcs_kstatus, SDBC_ESIZE, itmp);
 284                 rc = SDBC_EENABLEFAIL;
 285                 goto out;
 286         }
 287 
 288         if (_sd_cache_config.magic != _SD_MAGIC) {
 289                 rc = SDBC_EMAGIC;
 290                 goto out;
 291         }
 292 
 293         sdbc_use_dmchain = (_sd_cache_config.reserved1 & CFG_USE_DMCHAIN);
 294         sdbc_static_cache =  (_sd_cache_config.reserved1 & CFG_STATIC_CACHE);
 295 
 296         _sdbc_nodeid_configure();
 297 
 298         if (_SD_SELF_HOST > nsc_max_nodeid ||
 299             _SD_MIRROR_HOST > nsc_max_nodeid) {
 300                 (void) spcs_s_inttostring((_SD_SELF_HOST > nsc_max_nodeid ?
 301                     _SD_SELF_HOST : _SD_MIRROR_HOST), itmp, sizeof (itmp), 0);
 302                 (void) spcs_s_inttostring(
 303                     nsc_max_nodeid, itmp2, sizeof (itmp2), 0);
 304                 spcs_s_add(spcs_kstatus, SDBC_EINVHOSTID, itmp, itmp2);
 305                 rc = SDBC_EENABLEFAIL;
 306                 goto out;
 307         }
 308 
 309 
 310         if (_SD_SELF_HOST == _SD_MIRROR_HOST) {
 311                 (void) spcs_s_inttostring(
 312                     _SD_SELF_HOST, itmp, sizeof (itmp), 0);
 313                 (void) spcs_s_inttostring(
 314                     _SD_MIRROR_HOST, itmp2, sizeof (itmp2), 0);
 315                 spcs_s_add(spcs_kstatus, SDBC_ENOTSAME, itmp, itmp2);
 316                 rc = SDBC_EENABLEFAIL;
 317                 goto out;
 318         }
 319 
 320         /* initialize the safestore modules */
 321         sst_init();
 322 
 323         /* figure out which kind of safestore we need to use */
 324         ss_type = sdbc_determine_safestore();
 325 
 326 tryss:
 327         /* open and configure the safestore module */
 328         if ((sdbc_safestore = sst_open(ss_type, 0)) == NULL) {
 329                 cmn_err(CE_WARN, "!cannot open safestore module for type %x",
 330                     ss_type);
 331                 rc = SDBC_EENABLEFAIL;
 332                 goto out;
 333         } else {
 334                 sd_setup_ssconfig();
 335                 if (SSOP_CONFIGURE(sdbc_safestore, &safestore_config,
 336                     spcs_kstatus)) {
 337                         cmn_err(CE_WARN,
 338                             "!cannot configure safestore module for type %x",
 339                             ss_type);
 340                         (void) sst_close(sdbc_safestore);
 341 
 342                         /* try ram if possible, otherwise return */
 343                         if ((ss_type & (SS_M_RAM | SS_T_NONE)) ==
 344                             (SS_M_RAM | SS_T_NONE)) {
 345                                 rc = SDBC_EENABLEFAIL;
 346                                 goto out;
 347                         }
 348 
 349                         ss_type = (SS_M_RAM | SS_T_NONE);
 350                         goto tryss;
 351                 }
 352         }
 353 
 354         if (SAFESTORE_LOCAL(sdbc_safestore))
 355                 _SD_MIRROR_HOST = _SD_NO_HOST;
 356 
 357         ASSERT(safestore_config.ssc_ss_psize <= UINT16_MAX); /* LINTED */
 358         _sd_net_config.sn_psize = safestore_config.ssc_ss_psize;
 359 
 360 
 361         _sd_net_config.sn_csize =
 362             _sd_cache_config.cache_mem[_SD_NO_NET] * MEGABYTE;
 363         _sd_net_config.sn_cpages =
 364             _sd_net_config.sn_csize / BLK_SIZE(1);
 365 
 366         _sd_net_config.sn_configured = 1;
 367         cache_bytes = _sd_net_config.sn_cpages * BLK_SIZE(1);
 368 
 369         if (_sdbc_memtype_configure()) {
 370                 rc = EINVAL;
 371                 goto out;
 372         }
 373 
 374         if ((rc = _sdbc_iobuf_configure(_sd_cache_config.iobuf))) {
 375                 if (rc == -1) {
 376                         rc = SDBC_ENOIOBMEM;
 377                         goto out;
 378                 }
 379                 if (rc == -2) {
 380                         rc = SDBC_ENOIOBCB;
 381                         goto out;
 382                 }
 383 
 384         }
 385 
 386         if (_sdbc_handles_configure()) {
 387                 rc = SDBC_ENOHANDLEMEM;
 388                 goto out;
 389         }
 390 
 391         _sd_cache_dem_cnt = 0;
 392 
 393 
 394         /*
 395          * nvmem support:
 396          * if the cache did not shutdown properly we mark it as dirty.
 397          * this must be done before _sdbc_cache_configure() so it can
 398          * refresh sd_info_mem and sd_file_mem from nvmem if necsssary,
 399          * and before _sdbc_ft_configure() so the ft thread will do a recovery.
 400          *
 401          */
 402         if (SAFESTORE_RECOVERY(sdbc_safestore)) {
 403                 _sdbc_set_warm_start();
 404                 _sdbc_ft_hold_io = 1;
 405                 cmn_err(CE_WARN,
 406                     "!sdbc(_sdbc_configure) cache marked dirty after"
 407                     " incomplete shutdown");
 408         }
 409 
 410         if ((rc = _sdbc_cache_configure(cache_bytes / BLK_SIZE(1),
 411             spcs_kstatus))) {
 412                 goto out;
 413         }
 414 
 415 
 416         /* ST_ALERT trace buffer */
 417         if (_sdbc_tr_configure(-1 /* SDT_INV_CD */) != 0) {
 418                 rc = EINVAL;
 419                 goto out;
 420         }
 421 
 422         if (_sdbc_thread_configure()) {
 423                 rc = SDBC_EFLUSHTHRD;
 424                 goto out;
 425         }
 426 
 427         if (_sdbc_flush_configure()) {
 428                 rc = EINVAL;
 429                 goto out;
 430         }
 431 
 432         if (rc = _sdbc_dealloc_configure_dm()) {
 433                 goto out;
 434         }
 435 
 436         if (_sd_cache_config.test_demons)
 437                 if (_sdbc_tdaemon_configure(_sd_cache_config.test_demons)) {
 438                         rc = EINVAL;
 439                         goto out;
 440                 }
 441 
 442 
 443         _sd_cache_initialized = 1;
 444 
 445         sdbc_power = nsc_register_power("sdbc", _sdbc_power_def);
 446 
 447         if (_sdbc_ft_configure() != 0) {
 448                 rc = EINVAL;
 449                 goto out;
 450         }
 451 
 452         /*
 453          * try to control the race between the ft thread
 454          * and threads that will open the devices that the ft thread
 455          * may be recovering.  this synchronizing with the ft thread
 456          * prevents sd_cadmin from returning until ft has opened
 457          * the recovery devices, so if other apps wait for sd_cadmin
 458          * to complete the race is prevented.
 459          */
 460         mutex_enter(&_sdbc_ft_hold_io_lk);
 461         while (_sdbc_ft_hold_io) {
 462                 cv_wait(&_sdbc_ft_hold_io_cv, &_sdbc_ft_hold_io_lk);
 463         }
 464 
 465         io = nsc_register_io("sdbc", NSC_SDBC_ID|NSC_FILTER,
 466             _sd_sdbc_def);
 467 
 468         if (io) sdbc_io = io;
 469 
 470         mutex_exit(&_sdbc_ft_hold_io_lk);
 471 
 472 #ifdef DEBUG
 473         cmn_err(CE_NOTE, "!sd_config: Cache has been configured");
 474 #endif
 475 
 476         rc = 0;
 477 
 478 out:
 479         return (rc);
 480 }
 481 
 482 /*
 483  * _sdbc_deconfigure - Put the cache back to the unconfigured state. Release
 484  * any memory we allocated as part of the configuration process (but not the
 485  * load/init process).  Put globals back to unconfigured state and shut down
 486  * any processes/threads we have running.
 487  *
 488  * Since the cache has loaded we know that global lock/sv's are present and
 489  * we can use them to produce an orderly deconfiguration.
 490  *
 491  * NOTE: this routine and its callee should always be capable of reversing
 492  * the effects of _sdbc_configure no matter what partially configured
 493  * state might be present.
 494  *
 495  */
 496 int
 497 _sdbc_deconfigure(spcs_s_info_t spcs_kstatus)
 498 {
 499         int i;
 500         _sd_cd_info_t *cdi;
 501         int rc;
 502         int pinneddata = 0;
 503         uint_t saved_hint;
 504 
 505         ASSERT(MUTEX_HELD(&_sdbc_config_lock));
 506 
 507 #ifdef DEBUG
 508         cmn_err(CE_NOTE, "!SD cache being deconfigured.");
 509 #endif
 510 
 511         /* check if there is pinned data and our mirror is down */
 512         if (_sd_cache_files && _sd_is_mirror_down()) {
 513                 for (i = 0; i < sdbc_max_devs; i++) {
 514                         cdi = &(_sd_cache_files[i]);
 515                         if (cdi->cd_info == NULL)
 516                                 continue;
 517                         /*
 518                          * if (!(cdi->cd_info->sh_failed))
 519                          *      continue;
 520                          */
 521                         if (!(_SD_CD_ALL_WRITES(i)))
 522                                 continue;
 523                         spcs_s_add(spcs_kstatus, SDBC_EPINNED,
 524                             cdi->cd_info->sh_filename);
 525                         rc = SDBC_EDISABLEFAIL;
 526                         goto out;
 527                 }
 528         }
 529 
 530         /* remember hint setting for restoration in case shutdown fails */
 531         (void) _sd_get_node_hint(&saved_hint);
 532 
 533         (void) _sd_set_node_hint(NSC_FORCED_WRTHRU);
 534 
 535 
 536         /* TODO - there is a possible race between deconfig and power hits... */
 537 
 538         if (sdbc_power)
 539                 (void) nsc_unregister_power(sdbc_power);
 540 
 541 
 542         if (sdbc_io) {
 543                 rc = nsc_unregister_io(sdbc_io, NSC_PCATCH);
 544                 if (rc == 0)
 545                         sdbc_io = NULL;
 546                 else {
 547                         if (rc == EUSERS)
 548                                 spcs_s_add(spcs_kstatus, SDBC_EABUFS);
 549 
 550                         spcs_s_add(spcs_kstatus, SDBC_EUNREG);
 551 
 552                         /* Re-register-power if it was register before. */
 553                         if (sdbc_power) {
 554                                 sdbc_power = nsc_register_power("sdbc",
 555                                     _sdbc_power_def);
 556                         }
 557 
 558                         /* Remove NSC_FORCED_WRTHRU if we set it */
 559                         (void) _sd_clear_node_hint(
 560                             (~saved_hint) & _SD_HINT_MASK);
 561 
 562                         rc = SDBC_EDISABLEFAIL;
 563                         goto out;
 564                 }
 565         }
 566 
 567         sdbc_power = NULL;
 568 
 569 #if defined(_SD_FAULT_RES)
 570         _sd_remote_disable(0);  /* notify mirror to forced_wrthru */
 571 #endif
 572         /*
 573          * close devices, deconfigure processes, wait for exits
 574          */
 575         _sdbc_tdaemon_deconfigure();
 576 
 577         if (_sd_cache_files) {
 578                 for (i = 0; i < sdbc_max_devs; i++) {
 579                         if (FILE_OPENED(i) && ((rc = _sd_close(i)) > 0)) {
 580                                 cmn_err(CE_WARN, "!sdbc(_sd_deconfigure)"
 581                                     " %d not closed (%d)\n", i, rc);
 582                         }
 583                 }
 584         }
 585 
 586         /*
 587          * look for pinned data
 588          * TODO sort this out for multinode systems.
 589          * cannot shutdown with pinned data on multinode.
 590          * the state of pinned data should be determined in
 591          * the close operation.
 592          */
 593         if (_sd_cache_files) {
 594                 for (i = 0; i < sdbc_max_devs; i++) {
 595                         cdi = &(_sd_cache_files[i]);
 596                         if (cdi->cd_info == NULL)
 597                                 continue;
 598                         /*
 599                          * if (!(cdi->cd_info->sh_failed))
 600                          *      continue;
 601                          */
 602                         if (!(_SD_CD_ALL_WRITES(i)))
 603                                 continue;
 604                         cmn_err(CE_WARN,
 605                             "!sdbc(_sd_deconfigure) Pinned Data on cd %d(%s)",
 606                             i, cdi->cd_info->sh_filename);
 607                         pinneddata++;
 608                 }
 609         }
 610 
 611         _sd_cache_initialized = 0;
 612 
 613         _sdbc_ft_deconfigure();
 614 
 615         _sdbc_flush_deconfigure();
 616         _sdbc_thread_deconfigure();
 617 
 618         mutex_enter(&_sd_cache_lock);
 619 
 620         while (_sd_cache_dem_cnt > 0) {
 621                 mutex_exit(&_sd_cache_lock);
 622                 (void) nsc_delay_sig(HZ/2);
 623                 mutex_enter(&_sd_cache_lock);
 624         }
 625         mutex_exit(&_sd_cache_lock);
 626 
 627         /*
 628          * remove all dynamically allocated cache data memory
 629          * there should be no i/o at this point
 630          */
 631         _sdbc_dealloc_deconfigure_dm();
 632         /*
 633          * At this point no thread of control should be active in the cache
 634          * but us (unless they are blocked on the config lock).
 635          */
 636 
 637 
 638 #if defined(_SD_FAULT_RES)
 639         _sd_remote_disable(1);  /* notify mirror I/O shutdown complete */
 640 #endif
 641 
 642 #define KEEP_TRACES     0       /* set to 1 keep traces after deconfig */
 643 #if !KEEP_TRACES
 644         /*
 645          * This needs to happen before we unregister the memory.
 646          */
 647         _sdbc_tr_deconfigure();
 648 #endif
 649 
 650 
 651         /* delete/free hash table, cache blocks, etc */
 652         _sdbc_cache_deconfigure();
 653 
 654         _sdbc_handles_deconfigure();
 655 
 656         _sdbc_iobuf_deconfigure();
 657 
 658 #if !KEEP_TRACES
 659         if (!_sdbc_memtype_deconfigure_delayed)
 660                 _sdbc_memtype_deconfigure();
 661 #else
 662         _sdbc_memtype_deconfigure_delayed = 1;
 663 #endif
 664 
 665         /*
 666          * Call ss deconfig(),
 667          * check for valid pointer in case _sdbc_configure()
 668          * failed before safestrore system was initialized.
 669          */
 670         if (sdbc_safestore)
 671                 SSOP_DECONFIGURE(sdbc_safestore, pinneddata);
 672 
 673         /* tear down safestore system */
 674         sst_deinit();
 675 
 676         _sdbc_nodeid_deconfigure();
 677 
 678         bzero(&_sd_cache_config, sizeof (_sd_cache_param_t));
 679 
 680         _SD_SELF_HOST = _SD_MIRROR_HOST = _SD_NO_HOST;
 681         _SD_NETS = 0;
 682         _sd_cblock_shift = 0;
 683         _sd_node_hint = 0;
 684 
 685 #ifdef DEBUG
 686         cmn_err(CE_NOTE, "!SD cache deconfigured.");
 687 #endif
 688 
 689         rc = 0;
 690 
 691 out:
 692         return (rc);
 693 }
 694 
 695 
 696 
 697 static int
 698 find_low_bit(int mask, int start)
 699 {
 700         for (; start < 32; start++)
 701                 if ((mask & (1 << start)))
 702                         break;
 703 
 704         return (start);
 705 }
 706 
 707 int
 708 get_high_bit(int size)
 709 {
 710         int lowbit;
 711         int newblk = size;
 712         int highbit = -1;
 713         int next_high = 0;
 714 
 715         while ((lowbit = find_low_bit(newblk, 0)) != 32) {
 716                 if (highbit >= 0) next_high = 1;
 717                 highbit = lowbit;
 718                 newblk &= ~(1 << highbit);
 719         }
 720 
 721         if (highbit <= 0) {
 722                 cmn_err(CE_WARN,
 723                     "!sdbc(get_high_bit) invalid block size %x\n", size);
 724                 return (-1);
 725         }
 726 
 727         if (next_high) highbit++;
 728 
 729         return (highbit);
 730 }
 731 
 732 
 733 int
 734 _sd_fill_pattern(caddr_t addr, uint_t pat, uint_t size)
 735 {
 736         caddr_t fmt_page;
 737         int i, page_size;
 738 
 739         page_size = (int)ptob(1);
 740 
 741         if ((fmt_page = (caddr_t)nsc_kmem_alloc(ptob(1),
 742             KM_SLEEP, sdbc_local_mem)) == NULL) {
 743                 cmn_err(CE_WARN, "!sdbc(_sd_fill pattern) no more memory");
 744                 return (-1);
 745         }
 746         for (i = 0; i < page_size; i += 4)
 747                 *(int *)(void *)(fmt_page + i) = pat;
 748 
 749         while (size >= page_size) {
 750                 bcopy(fmt_page, addr, ptob(1));
 751                 addr += page_size;
 752                 size -= page_size;
 753         }
 754         nsc_kmem_free(fmt_page, page_size);
 755         return (0);
 756 }
 757 
 758 
 759 /*
 760  * _sdbc_nodeid_deconfigure - merely a place holder until
 761  * such time as there is something to be undone w.r.t.
 762  * _sdbc_nodeid_configure.
 763  *
 764  */
 765 static void
 766 _sdbc_nodeid_deconfigure(void)
 767 {
 768         /* My but we're quick */
 769 }
 770 
 771 /*
 772  * _sdbc_nodeid_configure - configure the nodeid's we need to connect
 773  * to any other nodes in the network.
 774  *
 775  */
 776 void
 777 _sdbc_nodeid_configure(void)
 778 {
 779 
 780         if (_sd_cache_config.num_nodes == 0) {
 781                 _sd_nodes_configured = 1;
 782         } else {
 783                 _sd_nodes_configured = _sd_cache_config.num_nodes;
 784         }
 785 
 786         _SD_SELF_HOST   = nsc_node_id();
 787         _SD_MIRROR_HOST = _sd_cache_config.mirror_host;
 788 }
 789 
 790 #define STACK_SIZE      (32*1024)
 791 #define num_spin 0
 792 nstset_t *_sd_ioset;
 793 
 794 /*
 795  * _sdbc_thread_deconfigure - cache is being deconfigure, stop any
 796  * thread activity.
 797  *
 798  */
 799 static void
 800 _sdbc_thread_deconfigure(void)
 801 {
 802         ASSERT(MUTEX_HELD(&_sdbc_config_lock));
 803         nst_destroy(_sd_ioset);
 804         _sd_ioset = NULL;
 805 }
 806 
 807 /*
 808  * _sdbc_thread_configure - cache is being configured, initialize the
 809  * threads we need for flushing dirty cds.
 810  *
 811  */
 812 static int
 813 _sdbc_thread_configure(void)
 814 {
 815         ASSERT(MUTEX_HELD(&_sdbc_config_lock));
 816 
 817         if (!_sd_ioset)
 818                 _sd_ioset = nst_init("sd_thr", _sd_cache_config.threads);
 819 
 820         if (!_sd_ioset)
 821                 return (EINVAL);
 822 
 823         return (0);
 824 }
 825 
 826 int
 827 _sdbc_get_config(_sdbc_config_t *config_info)
 828 {
 829         int i;
 830 
 831         config_info->enabled = _sd_cache_initialized;
 832         config_info->magic = _SD_MAGIC;
 833         for (i = 0; i < CACHE_MEM_PAD; i++) {
 834                 config_info->cache_mem[i] = _sd_cache_config.cache_mem[i];
 835         }
 836         config_info->threads = _sd_cache_config.threads;
 837 
 838         return (0);
 839 }