1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013 David Hoeppner. All rights reserved.
  25  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  26  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  27  */
  28 
  29 /*
  30  * Display kernel statistics
  31  *
  32  * This is a reimplementation of the perl kstat command originally found
  33  * under usr/src/cmd/kstat/kstat.pl
  34  *
  35  * Incompatibilities:
  36  *      - perl regular expressions replaced with extended REs bracketed by '/'
  37  *
  38  * Flags added:
  39  *      -C      similar to the -p option but value is separated by a colon
  40  *      -h      display help
  41  *      -j      json format
  42  */
  43 
  44 #include <assert.h>
  45 #include <ctype.h>
  46 #include <errno.h>
  47 #include <kstat.h>
  48 #include <langinfo.h>
  49 #include <libgen.h>
  50 #include <limits.h>
  51 #include <locale.h>
  52 #include <signal.h>
  53 #include <stddef.h>
  54 #include <stdio.h>
  55 #include <stdlib.h>
  56 #include <string.h>
  57 #include <strings.h>
  58 #include <time.h>
  59 #include <unistd.h>
  60 #include <sys/list.h>
  61 #include <sys/time.h>
  62 #include <sys/types.h>
  63 
  64 #include "kstat.h"
  65 #include "statcommon.h"
  66 
  67 char    *cmdname = "kstat";     /* Name of this command */
  68 int     caught_cont = 0;        /* Have caught a SIGCONT */
  69 
  70 static uint_t   g_timestamp_fmt = NODATE;
  71 
  72 /* Helper flag - header was printed already? */
  73 static boolean_t g_headerflg;
  74 
  75 /* Saved command line options */
  76 static boolean_t g_cflg = B_FALSE;
  77 static boolean_t g_jflg = B_FALSE;
  78 static boolean_t g_lflg = B_FALSE;
  79 static boolean_t g_pflg = B_FALSE;
  80 static boolean_t g_qflg = B_FALSE;
  81 static ks_pattern_t     g_ks_class = {"*", 0};
  82 
  83 /* Return zero if a selector did match */
  84 static int      g_matched = 1;
  85 
  86 /* Sorted list of kstat instances */
  87 static list_t   instances_list;
  88 static list_t   selector_list;
  89 
  90 int
  91 main(int argc, char **argv)
  92 {
  93         ks_selector_t   *nselector;
  94         ks_selector_t   *uselector;
  95         kstat_ctl_t     *kc;
  96         hrtime_t        start_n;
  97         hrtime_t        period_n;
  98         boolean_t       errflg = B_FALSE;
  99         boolean_t       nselflg = B_FALSE;
 100         boolean_t       uselflg = B_FALSE;
 101         char            *q;
 102         int             count = 1;
 103         int             infinite_cycles = 0;
 104         int             interval = 0;
 105         int             n = 0;
 106         int             c, m, tmp;
 107 
 108         (void) setlocale(LC_ALL, "");
 109 #if !defined(TEXT_DOMAIN)               /* Should be defined by cc -D */
 110 #define TEXT_DOMAIN "SYS_TEST"          /* Use this only if it wasn't */
 111 #endif
 112         (void) textdomain(TEXT_DOMAIN);
 113 
 114         /*
 115          * Create the selector list and a dummy default selector to match
 116          * everything. While we process the cmdline options we will add
 117          * selectors to this list.
 118          */
 119         list_create(&selector_list, sizeof (ks_selector_t),
 120             offsetof(ks_selector_t, ks_next));
 121 
 122         nselector = new_selector();
 123 
 124         /*
 125          * Parse named command line arguments.
 126          */
 127         while ((c = getopt(argc, argv, "h?CqjlpT:m:i:n:s:c:")) != EOF)
 128                 switch (c) {
 129                 case 'h':
 130                 case '?':
 131                         usage();
 132                         exit(0);
 133                         break;
 134                 case 'C':
 135                         g_pflg = g_cflg = B_TRUE;
 136                         break;
 137                 case 'q':
 138                         g_qflg = B_TRUE;
 139                         break;
 140                 case 'j':
 141                         g_jflg = B_TRUE;
 142                         break;
 143                 case 'l':
 144                         g_pflg = g_lflg = B_TRUE;
 145                         break;
 146                 case 'p':
 147                         g_pflg = B_TRUE;
 148                         break;
 149                 case 'T':
 150                         switch (*optarg) {
 151                         case 'd':
 152                                 g_timestamp_fmt = DDATE;
 153                                 break;
 154                         case 'u':
 155                                 g_timestamp_fmt = UDATE;
 156                                 break;
 157                         default:
 158                                 errflg = B_TRUE;
 159                         }
 160                         break;
 161                 case 'm':
 162                         nselflg = B_TRUE;
 163                         nselector->ks_module.pstr =
 164                             (char *)ks_safe_strdup(optarg);
 165                         break;
 166                 case 'i':
 167                         nselflg = B_TRUE;
 168                         nselector->ks_instance.pstr =
 169                             (char *)ks_safe_strdup(optarg);
 170                         break;
 171                 case 'n':
 172                         nselflg = B_TRUE;
 173                         nselector->ks_name.pstr =
 174                             (char *)ks_safe_strdup(optarg);
 175                         break;
 176                 case 's':
 177                         nselflg = B_TRUE;
 178                         nselector->ks_statistic.pstr =
 179                             (char *)ks_safe_strdup(optarg);
 180                         break;
 181                 case 'c':
 182                         g_ks_class.pstr =
 183                             (char *)ks_safe_strdup(optarg);
 184                         break;
 185                 default:
 186                         errflg = B_TRUE;
 187                         break;
 188                 }
 189 
 190         if (g_qflg && (g_jflg || g_pflg)) {
 191                 (void) fprintf(stderr, gettext(
 192                     "-q and -lpj are mutually exclusive\n"));
 193                 errflg = B_TRUE;
 194         }
 195 
 196         if (errflg) {
 197                 usage();
 198                 exit(2);
 199         }
 200 
 201         argc -= optind;
 202         argv += optind;
 203 
 204         /*
 205          * Consume the rest of the command line. Parsing the
 206          * unnamed command line arguments.
 207          */
 208         while (argc--) {
 209                 errno = 0;
 210                 tmp = strtoul(*argv, &q, 10);
 211                 if (tmp == ULONG_MAX && errno == ERANGE) {
 212                         if (n == 0) {
 213                                 (void) fprintf(stderr, gettext(
 214                                     "Interval is too large\n"));
 215                         } else if (n == 1) {
 216                                 (void) fprintf(stderr, gettext(
 217                                     "Count is too large\n"));
 218                         }
 219                         usage();
 220                         exit(2);
 221                 }
 222 
 223                 if (errno != 0 || *q != '\0') {
 224                         m = 0;
 225                         uselector = new_selector();
 226                         while ((q = (char *)strsep(argv, ":")) != NULL) {
 227                                 m++;
 228                                 if (m > 4) {
 229                                         free(uselector);
 230                                         usage();
 231                                         exit(2);
 232                                 }
 233 
 234                                 if (*q != '\0') {
 235                                         switch (m) {
 236                                         case 1:
 237                                                 uselector->ks_module.pstr =
 238                                                     (char *)ks_safe_strdup(q);
 239                                                 break;
 240                                         case 2:
 241                                                 uselector->ks_instance.pstr =
 242                                                     (char *)ks_safe_strdup(q);
 243                                                 break;
 244                                         case 3:
 245                                                 uselector->ks_name.pstr =
 246                                                     (char *)ks_safe_strdup(q);
 247                                                 break;
 248                                         case 4:
 249                                                 uselector->ks_statistic.pstr =
 250                                                     (char *)ks_safe_strdup(q);
 251                                                 break;
 252                                         default:
 253                                                 assert(B_FALSE);
 254                                         }
 255                                 }
 256                         }
 257 
 258                         uselflg = B_TRUE;
 259                         list_insert_tail(&selector_list, uselector);
 260                 } else {
 261                         if (tmp < 1) {
 262                                 if (n == 0) {
 263                                         (void) fprintf(stderr, gettext(
 264                                             "Interval must be an "
 265                                             "integer >= 1"));
 266                                 } else if (n == 1) {
 267                                         (void) fprintf(stderr, gettext(
 268                                             "Count must be an integer >= 1"));
 269                                 }
 270                                 usage();
 271                                 exit(2);
 272                         } else {
 273                                 if (n == 0) {
 274                                         interval = tmp;
 275                                         count = -1;
 276                                 } else if (n == 1) {
 277                                         count = tmp;
 278                                 } else {
 279                                         usage();
 280                                         exit(2);
 281                                 }
 282                         }
 283                         n++;
 284                 }
 285                 argv++;
 286         }
 287 
 288         /*
 289          * Check if we founded a named selector on the cmdline.
 290          */
 291         if (uselflg) {
 292                 if (nselflg) {
 293                         (void) fprintf(stderr, gettext(
 294                             "[module[:instance[:name[:statistic]]]] and "
 295                             "-m -i -n -s are mutually exclusive"));
 296                         usage();
 297                         exit(2);
 298                 } else {
 299                         free(nselector);
 300                 }
 301         } else {
 302                 list_insert_tail(&selector_list, nselector);
 303         }
 304 
 305         assert(!list_is_empty(&selector_list));
 306 
 307         list_create(&instances_list, sizeof (ks_instance_t),
 308             offsetof(ks_instance_t, ks_next));
 309 
 310         while ((kc = kstat_open()) == NULL) {
 311                 if (errno == EAGAIN) {
 312                         (void) poll(NULL, 0, 200);
 313                 } else {
 314                         perror("kstat_open");
 315                         exit(3);
 316                 }
 317         }
 318 
 319         if (count > 1) {
 320                 if (signal(SIGCONT, cont_handler) == SIG_ERR) {
 321                         (void) fprintf(stderr, gettext(
 322                             "signal failed"));
 323                         exit(3);
 324                 }
 325         }
 326 
 327         period_n = (hrtime_t)interval * NANOSEC;
 328         start_n = gethrtime();
 329 
 330         while (count == -1 || count-- > 0) {
 331                 ks_instances_read(kc);
 332                 ks_instances_print();
 333 
 334                 if (interval && count) {
 335                         ks_sleep_until(&start_n, period_n, infinite_cycles,
 336                             &caught_cont);
 337                         (void) kstat_chain_update(kc);
 338                         (void) putchar('\n');
 339                 }
 340         }
 341 
 342         (void) kstat_close(kc);
 343 
 344         return (g_matched);
 345 }
 346 
 347 /*
 348  * Print usage.
 349  */
 350 static void
 351 usage(void)
 352 {
 353         (void) fprintf(stderr, gettext(
 354             "Usage:\n"
 355             "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
 356             "      [ -m module ] [ -i instance ] [ -n name ] [ -s statistic ]\n"
 357             "      [ interval [ count ] ]\n"
 358             "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
 359             "      [ module[:instance[:name[:statistic]]] ... ]\n"
 360             "      [ interval [ count ] ]\n"));
 361 }
 362 
 363 /*
 364  * Sort compare function.
 365  */
 366 static int
 367 compare_instances(ks_instance_t *l_arg, ks_instance_t *r_arg)
 368 {
 369         int     rval;
 370 
 371         rval = strcasecmp(l_arg->ks_module, r_arg->ks_module);
 372         if (rval == 0) {
 373                 if (l_arg->ks_instance == r_arg->ks_instance) {
 374                         return (strcasecmp(l_arg->ks_name, r_arg->ks_name));
 375                 } else if (l_arg->ks_instance < r_arg->ks_instance) {
 376                         return (-1);
 377                 } else {
 378                         return (1);
 379                 }
 380         } else {
 381                 return (rval);
 382         }
 383 }
 384 
 385 static char *
 386 ks_safe_strdup(char *str)
 387 {
 388         char    *ret;
 389 
 390         if (str == NULL) {
 391                 return (NULL);
 392         }
 393 
 394         while ((ret = strdup(str)) == NULL) {
 395                 if (errno == EAGAIN) {
 396                         (void) poll(NULL, 0, 200);
 397                 } else {
 398                         perror("strdup");
 399                         exit(3);
 400                 }
 401         }
 402 
 403         return (ret);
 404 }
 405 
 406 static void
 407 ks_sleep_until(hrtime_t *wakeup, hrtime_t interval, int forever,
 408     int *caught_cont)
 409 {
 410         hrtime_t        now, pause, pause_left;
 411         struct timespec pause_tv;
 412         int             status;
 413 
 414         now = gethrtime();
 415         pause = *wakeup + interval - now;
 416 
 417         if (pause <= 0 || pause < (interval / 4)) {
 418                 if (forever || *caught_cont) {
 419                         *wakeup = now + interval;
 420                         pause = interval;
 421                 } else {
 422                         pause = interval / 2;
 423                         *wakeup += interval;
 424                 }
 425         } else {
 426                 *wakeup += interval;
 427         }
 428 
 429         if (pause < 1000) {
 430                 return;
 431         }
 432 
 433         pause_left = pause;
 434         do {
 435                 pause_tv.tv_sec = pause_left / NANOSEC;
 436                 pause_tv.tv_nsec = pause_left % NANOSEC;
 437                 status = nanosleep(&pause_tv, (struct timespec *)NULL);
 438                 if (status < 0) {
 439                         if (errno == EINTR) {
 440                                 now = gethrtime();
 441                                 pause_left = *wakeup - now;
 442                                 if (pause_left < 1000) {
 443                                         return;
 444                                 }
 445                         } else {
 446                                 perror("nanosleep");
 447                                 exit(3);
 448                         }
 449                 }
 450         } while (status != 0);
 451 }
 452 
 453 /*
 454  * Inserts an instance in the per selector list.
 455  */
 456 static void
 457 nvpair_insert(ks_instance_t *ksi, char *name, ks_value_t *value,
 458     uchar_t data_type)
 459 {
 460         ks_nvpair_t     *instance;
 461         ks_nvpair_t     *tmp;
 462 
 463         instance = (ks_nvpair_t *)malloc(sizeof (ks_nvpair_t));
 464         if (instance == NULL) {
 465                 perror("malloc");
 466                 exit(3);
 467         }
 468 
 469         (void) strlcpy(instance->name, name, KSTAT_STRLEN);
 470         (void) memcpy(&instance->value, value, sizeof (ks_value_t));
 471         instance->data_type = data_type;
 472 
 473         tmp = list_head(&ksi->ks_nvlist);
 474         while (tmp != NULL && strcasecmp(instance->name, tmp->name) > 0)
 475                 tmp = list_next(&ksi->ks_nvlist, tmp);
 476 
 477         list_insert_before(&ksi->ks_nvlist, tmp, instance);
 478 }
 479 
 480 /*
 481  * Allocates a new all-matching selector.
 482  */
 483 static ks_selector_t *
 484 new_selector(void)
 485 {
 486         ks_selector_t   *selector;
 487 
 488         selector = (ks_selector_t *)malloc(sizeof (ks_selector_t));
 489         if (selector == NULL) {
 490                 perror("malloc");
 491                 exit(3);
 492         }
 493 
 494         list_link_init(&selector->ks_next);
 495 
 496         selector->ks_module.pstr = "*";
 497         selector->ks_instance.pstr = "*";
 498         selector->ks_name.pstr = "*";
 499         selector->ks_statistic.pstr = "*";
 500 
 501         return (selector);
 502 }
 503 
 504 /*
 505  * This function was taken from the perl kstat module code - please
 506  * see for further comments there.
 507  */
 508 static kstat_raw_reader_t
 509 lookup_raw_kstat_fn(char *module, char *name)
 510 {
 511         char            key[KSTAT_STRLEN * 2];
 512         register char   *f, *t;
 513         int             n = 0;
 514 
 515         for (f = module, t = key; *f != '\0'; f++, t++) {
 516                 while (*f != '\0' && isdigit(*f))
 517                         f++;
 518                 *t = *f;
 519         }
 520         *t++ = ':';
 521 
 522         for (f = name; *f != '\0'; f++, t++) {
 523                 while (*f != '\0' && isdigit(*f))
 524                         f++;
 525                 *t = *f;
 526         }
 527         *t = '\0';
 528 
 529         while (ks_raw_lookup[n].fn != NULL) {
 530                 if (strncmp(ks_raw_lookup[n].name, key, strlen(key)) == 0)
 531                         return (ks_raw_lookup[n].fn);
 532                 n++;
 533         }
 534 
 535         return (0);
 536 }
 537 
 538 /*
 539  * Match a string against a shell glob or extended regular expression.
 540  */
 541 static boolean_t
 542 ks_match(const char *str, ks_pattern_t *pattern)
 543 {
 544         int     regcode;
 545         char    *regstr;
 546         char    *errbuf;
 547         size_t  bufsz;
 548 
 549         if (pattern->pstr != NULL && gmatch(pattern->pstr, "/*/") != 0) {
 550                 /* All regex patterns are strdup'd copies */
 551                 regstr = pattern->pstr + 1;
 552                 *(strrchr(regstr, '/')) = '\0';
 553 
 554                 regcode = regcomp(&pattern->preg, regstr,
 555                     REG_EXTENDED | REG_NOSUB);
 556                 if (regcode != 0) {
 557                         bufsz = regerror(regcode, NULL, NULL, 0);
 558                         if (bufsz != 0) {
 559                                 errbuf = malloc(bufsz);
 560                                 if (errbuf == NULL) {
 561                                         perror("malloc");
 562                                         exit(3);
 563                                 }
 564                                 (void) regerror(regcode, NULL, errbuf, bufsz);
 565                                 (void) fprintf(stderr, "kstat: %s\n", errbuf);
 566                         }
 567                         usage();
 568                         exit(2);
 569                 }
 570 
 571                 pattern->pstr = NULL;
 572         }
 573 
 574         if (pattern->pstr == NULL) {
 575                 return (regexec(&pattern->preg, str, 0, NULL, 0) == 0);
 576         }
 577 
 578         return ((gmatch(str, pattern->pstr) != 0));
 579 }
 580 
 581 /*
 582  * Iterate over all kernel statistics and save matches.
 583  */
 584 static void
 585 ks_instances_read(kstat_ctl_t *kc)
 586 {
 587         kstat_raw_reader_t save_raw = NULL;
 588         kid_t           id;
 589         ks_selector_t   *selector;
 590         ks_instance_t   *ksi;
 591         ks_instance_t   *tmp;
 592         kstat_t         *kp;
 593         boolean_t       skip;
 594 
 595         for (kp = kc->kc_chain; kp != NULL; kp = kp->ks_next) {
 596                 /* Don't bother storing the kstat headers */
 597                 if (strncmp(kp->ks_name, "kstat_", 6) == 0) {
 598                         continue;
 599                 }
 600 
 601                 /* Don't bother storing raw stats we don't understand */
 602                 if (kp->ks_type == KSTAT_TYPE_RAW) {
 603                         save_raw = lookup_raw_kstat_fn(kp->ks_module,
 604                             kp->ks_name);
 605                         if (save_raw == NULL) {
 606 #ifdef REPORT_UNKNOWN
 607                                 (void) fprintf(stderr,
 608                                     "Unknown kstat type %s:%d:%s - "
 609                                     "%d of size %d\n", kp->ks_module,
 610                                     kp->ks_instance, kp->ks_name,
 611                                     kp->ks_ndata, kp->ks_data_size);
 612 #endif
 613                                 continue;
 614                         }
 615                 }
 616 
 617                 /*
 618                  * Iterate over the list of selectors and skip
 619                  * instances we dont want. We filter for statistics
 620                  * later, as we dont know them yet.
 621                  */
 622                 skip = B_TRUE;
 623                 selector = list_head(&selector_list);
 624                 while (selector != NULL) {
 625                         if (ks_match(kp->ks_module, &selector->ks_module) &&
 626                             ks_match(kp->ks_name, &selector->ks_name)) {
 627                                 skip = B_FALSE;
 628                                 break;
 629                         }
 630                         selector = list_next(&selector_list, selector);
 631                 }
 632 
 633                 if (skip) {
 634                         continue;
 635                 }
 636 
 637                 /*
 638                  * Allocate a new instance and fill in the values
 639                  * we know so far.
 640                  */
 641                 ksi = (ks_instance_t *)malloc(sizeof (ks_instance_t));
 642                 if (ksi == NULL) {
 643                         perror("malloc");
 644                         exit(3);
 645                 }
 646 
 647                 list_link_init(&ksi->ks_next);
 648 
 649                 (void) strlcpy(ksi->ks_module, kp->ks_module, KSTAT_STRLEN);
 650                 (void) strlcpy(ksi->ks_name, kp->ks_name, KSTAT_STRLEN);
 651                 (void) strlcpy(ksi->ks_class, kp->ks_class, KSTAT_STRLEN);
 652 
 653                 ksi->ks_instance = kp->ks_instance;
 654                 ksi->ks_snaptime = kp->ks_snaptime;
 655                 ksi->ks_type = kp->ks_type;
 656 
 657                 list_create(&ksi->ks_nvlist, sizeof (ks_nvpair_t),
 658                     offsetof(ks_nvpair_t, nv_next));
 659 
 660                 /* Insert this instance into a sorted list */
 661                 tmp = list_head(&instances_list);
 662                 while (tmp != NULL && compare_instances(ksi, tmp) > 0)
 663                         tmp = list_next(&instances_list, tmp);
 664 
 665                 list_insert_before(&instances_list, tmp, ksi);
 666 
 667                 /* Read the actual statistics */
 668                 id = kstat_read(kc, kp, NULL);
 669                 if (id == -1) {
 670 #ifdef REPORT_UNKNOWN
 671                         perror("kstat_read");
 672 #endif
 673                         continue;
 674                 }
 675 
 676                 SAVE_HRTIME_X(ksi, "crtime", kp->ks_crtime);
 677                 SAVE_HRTIME_X(ksi, "snaptime", kp->ks_snaptime);
 678                 if (g_pflg) {
 679                         SAVE_STRING_X(ksi, "class", kp->ks_class);
 680                 }
 681 
 682                 switch (kp->ks_type) {
 683                 case KSTAT_TYPE_RAW:
 684                         save_raw(kp, ksi);
 685                         break;
 686                 case KSTAT_TYPE_NAMED:
 687                         save_named(kp, ksi);
 688                         break;
 689                 case KSTAT_TYPE_INTR:
 690                         save_intr(kp, ksi);
 691                         break;
 692                 case KSTAT_TYPE_IO:
 693                         save_io(kp, ksi);
 694                         break;
 695                 case KSTAT_TYPE_TIMER:
 696                         save_timer(kp, ksi);
 697                         break;
 698                 default:
 699                         assert(B_FALSE); /* Invalid type */
 700                         break;
 701                 }
 702         }
 703 }
 704 
 705 /*
 706  * Print the value of a name-value pair.
 707  */
 708 static void
 709 ks_value_print(ks_nvpair_t *nvpair)
 710 {
 711         switch (nvpair->data_type) {
 712         case KSTAT_DATA_CHAR:
 713                 (void) fprintf(stdout, "%s", nvpair->value.c);
 714                 break;
 715         case KSTAT_DATA_INT32:
 716                 (void) fprintf(stdout, "%d", nvpair->value.i32);
 717                 break;
 718         case KSTAT_DATA_UINT32:
 719                 (void) fprintf(stdout, "%u", nvpair->value.ui32);
 720                 break;
 721         case KSTAT_DATA_INT64:
 722                 (void) fprintf(stdout, "%lld", nvpair->value.i64);
 723                 break;
 724         case KSTAT_DATA_UINT64:
 725                 (void) fprintf(stdout, "%llu", nvpair->value.ui64);
 726                 break;
 727         case KSTAT_DATA_STRING:
 728                 (void) fprintf(stdout, "%s", KSTAT_NAMED_STR_PTR(nvpair));
 729                 break;
 730         case KSTAT_DATA_HRTIME:
 731                 if (nvpair->value.ui64 == 0)
 732                         (void) fprintf(stdout, "0");
 733                 else
 734                         (void) fprintf(stdout, "%.9f",
 735                             nvpair->value.ui64 / 1000000000.0);
 736                 break;
 737         default:
 738                 assert(B_FALSE);
 739         }
 740 }
 741 
 742 /*
 743  * Print a single instance.
 744  */
 745 static void
 746 ks_instance_print(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 747 {
 748         if (g_headerflg) {
 749                 if (!g_pflg) {
 750                         (void) fprintf(stdout, DFLT_FMT,
 751                             ksi->ks_module, ksi->ks_instance,
 752                             ksi->ks_name, ksi->ks_class);
 753                 }
 754                 g_headerflg = B_FALSE;
 755         }
 756 
 757         if (g_pflg) {
 758                 (void) fprintf(stdout, KS_PFMT,
 759                     ksi->ks_module, ksi->ks_instance,
 760                     ksi->ks_name, nvpair->name);
 761                 if (!g_lflg) {
 762                         (void) putchar(g_cflg ? ':': '\t');
 763                         ks_value_print(nvpair);
 764                 }
 765         } else {
 766                 (void) fprintf(stdout, KS_DFMT, nvpair->name);
 767                 ks_value_print(nvpair);
 768         }
 769 
 770         (void) putchar('\n');
 771 }
 772 
 773 /*
 774  * Print a single instance in JSON format.
 775  */
 776 static void
 777 ks_instance_print_json(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 778 {
 779         if (g_headerflg) {
 780                 (void) fprintf(stdout, JSON_FMT,
 781                     ksi->ks_module, ksi->ks_instance,
 782                     ksi->ks_name, ksi->ks_class,
 783                     ksi->ks_type);
 784 
 785                 if (ksi->ks_snaptime == 0)
 786                         (void) fprintf(stdout, "\t\"snaptime\": 0,\n");
 787                 else
 788                         (void) fprintf(stdout, "\t\"snaptime\": %.9f,\n",
 789                             ksi->ks_snaptime / 1000000000.0);
 790 
 791                 (void) fprintf(stdout, "\t\"data\": {\n");
 792 
 793                 g_headerflg = B_FALSE;
 794         }
 795 
 796         (void) fprintf(stdout, KS_JFMT, nvpair->name);
 797         if (nvpair->data_type == KSTAT_DATA_STRING) {
 798                 (void) putchar('\"');
 799                 ks_value_print(nvpair);
 800                 (void) putchar('\"');
 801         } else {
 802                 ks_value_print(nvpair);
 803         }
 804         if (nvpair != list_tail(&ksi->ks_nvlist))
 805                 (void) putchar(',');
 806 
 807         (void) putchar('\n');
 808 }
 809 
 810 /*
 811  * Print all instances.
 812  */
 813 static void
 814 ks_instances_print(void)
 815 {
 816         ks_selector_t   *selector;
 817         ks_instance_t   *ksi, *ktmp;
 818         ks_nvpair_t     *nvpair, *ntmp;
 819         void            (*ks_print_fn)(ks_instance_t *, ks_nvpair_t *);
 820         char            *ks_number;
 821 
 822         if (g_timestamp_fmt != NODATE)
 823                 print_timestamp(g_timestamp_fmt);
 824 
 825         if (g_jflg) {
 826                 ks_print_fn = &ks_instance_print_json;
 827                 (void) putchar('[');
 828         } else {
 829                 ks_print_fn = &ks_instance_print;
 830         }
 831 
 832         /* Iterate over each selector */
 833         selector = list_head(&selector_list);
 834         while (selector != NULL) {
 835 
 836                 /* Iterate over each instance */
 837                 for (ksi = list_head(&instances_list); ksi != NULL;
 838                     ksi = list_next(&instances_list, ksi)) {
 839 
 840                         (void) asprintf(&ks_number, "%d", ksi->ks_instance);
 841                         if (!(ks_match(ksi->ks_module, &selector->ks_module) &&
 842                             ks_match(ksi->ks_name, &selector->ks_name) &&
 843                             ks_match(ks_number, &selector->ks_instance) &&
 844                             ks_match(ksi->ks_class, &g_ks_class))) {
 845                                 free(ks_number);
 846                                 continue;
 847                         }
 848 
 849                         free(ks_number);
 850 
 851                         /* Finally iterate over each statistic */
 852                         g_headerflg = B_TRUE;
 853                         for (nvpair = list_head(&ksi->ks_nvlist);
 854                             nvpair != NULL;
 855                             nvpair = list_next(&ksi->ks_nvlist, nvpair)) {
 856                                 if (!ks_match(nvpair->name,
 857                                     &selector->ks_statistic))
 858                                         continue;
 859 
 860                                 g_matched = 0;
 861                                 if (!g_qflg)
 862                                         (*ks_print_fn)(ksi, nvpair);
 863                         }
 864 
 865                         if (!g_headerflg) {
 866                                 if (g_jflg) {
 867                                         (void) fprintf(stdout, "\t}\n}");
 868                                         if (ksi != list_tail(&instances_list))
 869                                                 (void) putchar(',');
 870                                 } else if (!g_pflg) {
 871                                         (void) putchar('\n');
 872                                 }
 873                         }
 874                 }
 875 
 876                 selector = list_next(&selector_list, selector);
 877         }
 878 
 879         if (g_jflg)
 880                 (void) fprintf(stdout, "]\n");
 881 
 882         (void) fflush(stdout);
 883 
 884         /* Free the instances list */
 885         ksi = list_head(&instances_list);
 886         while (ksi != NULL) {
 887                 nvpair = list_head(&ksi->ks_nvlist);
 888                 while (nvpair != NULL) {
 889                         ntmp = nvpair;
 890                         nvpair = list_next(&ksi->ks_nvlist, nvpair);
 891                         list_remove(&ksi->ks_nvlist, ntmp);
 892                         if (ntmp->data_type == KSTAT_DATA_STRING)
 893                                 free(ntmp->value.str.addr.ptr);
 894                         free(ntmp);
 895                 }
 896 
 897                 ktmp = ksi;
 898                 ksi = list_next(&instances_list, ksi);
 899                 list_remove(&instances_list, ktmp);
 900                 list_destroy(&ktmp->ks_nvlist);
 901                 free(ktmp);
 902         }
 903 }
 904 
 905 static void
 906 save_cpu_stat(kstat_t *kp, ks_instance_t *ksi)
 907 {
 908         cpu_stat_t      *stat;
 909         cpu_sysinfo_t   *sysinfo;
 910         cpu_syswait_t   *syswait;
 911         cpu_vminfo_t    *vminfo;
 912 
 913         stat = (cpu_stat_t *)(kp->ks_data);
 914         sysinfo = &stat->cpu_sysinfo;
 915         syswait = &stat->cpu_syswait;
 916         vminfo  = &stat->cpu_vminfo;
 917 
 918         SAVE_UINT32_X(ksi, "idle", sysinfo->cpu[CPU_IDLE]);
 919         SAVE_UINT32_X(ksi, "user", sysinfo->cpu[CPU_USER]);
 920         SAVE_UINT32_X(ksi, "kernel", sysinfo->cpu[CPU_KERNEL]);
 921         SAVE_UINT32_X(ksi, "wait", sysinfo->cpu[CPU_WAIT]);
 922         SAVE_UINT32_X(ksi, "wait_io", sysinfo->wait[W_IO]);
 923         SAVE_UINT32_X(ksi, "wait_swap", sysinfo->wait[W_SWAP]);
 924         SAVE_UINT32_X(ksi, "wait_pio", sysinfo->wait[W_PIO]);
 925         SAVE_UINT32(ksi, sysinfo, bread);
 926         SAVE_UINT32(ksi, sysinfo, bwrite);
 927         SAVE_UINT32(ksi, sysinfo, lread);
 928         SAVE_UINT32(ksi, sysinfo, lwrite);
 929         SAVE_UINT32(ksi, sysinfo, phread);
 930         SAVE_UINT32(ksi, sysinfo, phwrite);
 931         SAVE_UINT32(ksi, sysinfo, pswitch);
 932         SAVE_UINT32(ksi, sysinfo, trap);
 933         SAVE_UINT32(ksi, sysinfo, intr);
 934         SAVE_UINT32(ksi, sysinfo, syscall);
 935         SAVE_UINT32(ksi, sysinfo, sysread);
 936         SAVE_UINT32(ksi, sysinfo, syswrite);
 937         SAVE_UINT32(ksi, sysinfo, sysfork);
 938         SAVE_UINT32(ksi, sysinfo, sysvfork);
 939         SAVE_UINT32(ksi, sysinfo, sysexec);
 940         SAVE_UINT32(ksi, sysinfo, readch);
 941         SAVE_UINT32(ksi, sysinfo, writech);
 942         SAVE_UINT32(ksi, sysinfo, rcvint);
 943         SAVE_UINT32(ksi, sysinfo, xmtint);
 944         SAVE_UINT32(ksi, sysinfo, mdmint);
 945         SAVE_UINT32(ksi, sysinfo, rawch);
 946         SAVE_UINT32(ksi, sysinfo, canch);
 947         SAVE_UINT32(ksi, sysinfo, outch);
 948         SAVE_UINT32(ksi, sysinfo, msg);
 949         SAVE_UINT32(ksi, sysinfo, sema);
 950         SAVE_UINT32(ksi, sysinfo, namei);
 951         SAVE_UINT32(ksi, sysinfo, ufsiget);
 952         SAVE_UINT32(ksi, sysinfo, ufsdirblk);
 953         SAVE_UINT32(ksi, sysinfo, ufsipage);
 954         SAVE_UINT32(ksi, sysinfo, ufsinopage);
 955         SAVE_UINT32(ksi, sysinfo, inodeovf);
 956         SAVE_UINT32(ksi, sysinfo, fileovf);
 957         SAVE_UINT32(ksi, sysinfo, procovf);
 958         SAVE_UINT32(ksi, sysinfo, intrthread);
 959         SAVE_UINT32(ksi, sysinfo, intrblk);
 960         SAVE_UINT32(ksi, sysinfo, idlethread);
 961         SAVE_UINT32(ksi, sysinfo, inv_swtch);
 962         SAVE_UINT32(ksi, sysinfo, nthreads);
 963         SAVE_UINT32(ksi, sysinfo, cpumigrate);
 964         SAVE_UINT32(ksi, sysinfo, xcalls);
 965         SAVE_UINT32(ksi, sysinfo, mutex_adenters);
 966         SAVE_UINT32(ksi, sysinfo, rw_rdfails);
 967         SAVE_UINT32(ksi, sysinfo, rw_wrfails);
 968         SAVE_UINT32(ksi, sysinfo, modload);
 969         SAVE_UINT32(ksi, sysinfo, modunload);
 970         SAVE_UINT32(ksi, sysinfo, bawrite);
 971 #ifdef  STATISTICS      /* see header file */
 972         SAVE_UINT32(ksi, sysinfo, rw_enters);
 973         SAVE_UINT32(ksi, sysinfo, win_uo_cnt);
 974         SAVE_UINT32(ksi, sysinfo, win_uu_cnt);
 975         SAVE_UINT32(ksi, sysinfo, win_so_cnt);
 976         SAVE_UINT32(ksi, sysinfo, win_su_cnt);
 977         SAVE_UINT32(ksi, sysinfo, win_suo_cnt);
 978 #endif
 979 
 980         SAVE_INT32(ksi, syswait, iowait);
 981         SAVE_INT32(ksi, syswait, swap);
 982         SAVE_INT32(ksi, syswait, physio);
 983 
 984         SAVE_UINT32(ksi, vminfo, pgrec);
 985         SAVE_UINT32(ksi, vminfo, pgfrec);
 986         SAVE_UINT32(ksi, vminfo, pgin);
 987         SAVE_UINT32(ksi, vminfo, pgpgin);
 988         SAVE_UINT32(ksi, vminfo, pgout);
 989         SAVE_UINT32(ksi, vminfo, pgpgout);
 990         SAVE_UINT32(ksi, vminfo, swapin);
 991         SAVE_UINT32(ksi, vminfo, pgswapin);
 992         SAVE_UINT32(ksi, vminfo, swapout);
 993         SAVE_UINT32(ksi, vminfo, pgswapout);
 994         SAVE_UINT32(ksi, vminfo, zfod);
 995         SAVE_UINT32(ksi, vminfo, dfree);
 996         SAVE_UINT32(ksi, vminfo, scan);
 997         SAVE_UINT32(ksi, vminfo, rev);
 998         SAVE_UINT32(ksi, vminfo, hat_fault);
 999         SAVE_UINT32(ksi, vminfo, as_fault);
1000         SAVE_UINT32(ksi, vminfo, maj_fault);
1001         SAVE_UINT32(ksi, vminfo, cow_fault);
1002         SAVE_UINT32(ksi, vminfo, prot_fault);
1003         SAVE_UINT32(ksi, vminfo, softlock);
1004         SAVE_UINT32(ksi, vminfo, kernel_asflt);
1005         SAVE_UINT32(ksi, vminfo, pgrrun);
1006         SAVE_UINT32(ksi, vminfo, execpgin);
1007         SAVE_UINT32(ksi, vminfo, execpgout);
1008         SAVE_UINT32(ksi, vminfo, execfree);
1009         SAVE_UINT32(ksi, vminfo, anonpgin);
1010         SAVE_UINT32(ksi, vminfo, anonpgout);
1011         SAVE_UINT32(ksi, vminfo, anonfree);
1012         SAVE_UINT32(ksi, vminfo, fspgin);
1013         SAVE_UINT32(ksi, vminfo, fspgout);
1014         SAVE_UINT32(ksi, vminfo, fsfree);
1015 }
1016 
1017 static void
1018 save_var(kstat_t *kp, ks_instance_t *ksi)
1019 {
1020         struct var      *var = (struct var *)(kp->ks_data);
1021 
1022         assert(kp->ks_data_size == sizeof (struct var));
1023 
1024         SAVE_INT32(ksi, var, v_buf);
1025         SAVE_INT32(ksi, var, v_call);
1026         SAVE_INT32(ksi, var, v_proc);
1027         SAVE_INT32(ksi, var, v_maxupttl);
1028         SAVE_INT32(ksi, var, v_nglobpris);
1029         SAVE_INT32(ksi, var, v_maxsyspri);
1030         SAVE_INT32(ksi, var, v_clist);
1031         SAVE_INT32(ksi, var, v_maxup);
1032         SAVE_INT32(ksi, var, v_hbuf);
1033         SAVE_INT32(ksi, var, v_hmask);
1034         SAVE_INT32(ksi, var, v_pbuf);
1035         SAVE_INT32(ksi, var, v_sptmap);
1036         SAVE_INT32(ksi, var, v_maxpmem);
1037         SAVE_INT32(ksi, var, v_autoup);
1038         SAVE_INT32(ksi, var, v_bufhwm);
1039 }
1040 
1041 static void
1042 save_ncstats(kstat_t *kp, ks_instance_t *ksi)
1043 {
1044         struct ncstats  *ncstats = (struct ncstats *)(kp->ks_data);
1045 
1046         assert(kp->ks_data_size == sizeof (struct ncstats));
1047 
1048         SAVE_INT32(ksi, ncstats, hits);
1049         SAVE_INT32(ksi, ncstats, misses);
1050         SAVE_INT32(ksi, ncstats, enters);
1051         SAVE_INT32(ksi, ncstats, dbl_enters);
1052         SAVE_INT32(ksi, ncstats, long_enter);
1053         SAVE_INT32(ksi, ncstats, long_look);
1054         SAVE_INT32(ksi, ncstats, move_to_front);
1055         SAVE_INT32(ksi, ncstats, purges);
1056 }
1057 
1058 static void
1059 save_sysinfo(kstat_t *kp, ks_instance_t *ksi)
1060 {
1061         sysinfo_t       *sysinfo = (sysinfo_t *)(kp->ks_data);
1062 
1063         assert(kp->ks_data_size == sizeof (sysinfo_t));
1064 
1065         SAVE_UINT32(ksi, sysinfo, updates);
1066         SAVE_UINT32(ksi, sysinfo, runque);
1067         SAVE_UINT32(ksi, sysinfo, runocc);
1068         SAVE_UINT32(ksi, sysinfo, swpque);
1069         SAVE_UINT32(ksi, sysinfo, swpocc);
1070         SAVE_UINT32(ksi, sysinfo, waiting);
1071 }
1072 
1073 static void
1074 save_vminfo(kstat_t *kp, ks_instance_t *ksi)
1075 {
1076         vminfo_t        *vminfo = (vminfo_t *)(kp->ks_data);
1077 
1078         assert(kp->ks_data_size == sizeof (vminfo_t));
1079 
1080         SAVE_UINT64(ksi, vminfo, freemem);
1081         SAVE_UINT64(ksi, vminfo, swap_resv);
1082         SAVE_UINT64(ksi, vminfo, swap_alloc);
1083         SAVE_UINT64(ksi, vminfo, swap_avail);
1084         SAVE_UINT64(ksi, vminfo, swap_free);
1085         SAVE_UINT64(ksi, vminfo, updates);
1086 }
1087 
1088 static void
1089 save_nfs(kstat_t *kp, ks_instance_t *ksi)
1090 {
1091         struct mntinfo_kstat *mntinfo = (struct mntinfo_kstat *)(kp->ks_data);
1092 
1093         assert(kp->ks_data_size == sizeof (struct mntinfo_kstat));
1094 
1095         SAVE_STRING(ksi, mntinfo, mik_proto);
1096         SAVE_UINT32(ksi, mntinfo, mik_vers);
1097         SAVE_UINT32(ksi, mntinfo, mik_flags);
1098         SAVE_UINT32(ksi, mntinfo, mik_secmod);
1099         SAVE_UINT32(ksi, mntinfo, mik_curread);
1100         SAVE_UINT32(ksi, mntinfo, mik_curwrite);
1101         SAVE_INT32(ksi, mntinfo, mik_timeo);
1102         SAVE_INT32(ksi, mntinfo, mik_retrans);
1103         SAVE_UINT32(ksi, mntinfo, mik_acregmin);
1104         SAVE_UINT32(ksi, mntinfo, mik_acregmax);
1105         SAVE_UINT32(ksi, mntinfo, mik_acdirmin);
1106         SAVE_UINT32(ksi, mntinfo, mik_acdirmax);
1107         SAVE_UINT32_X(ksi, "lookup_srtt", mntinfo->mik_timers[0].srtt);
1108         SAVE_UINT32_X(ksi, "lookup_deviate", mntinfo->mik_timers[0].deviate);
1109         SAVE_UINT32_X(ksi, "lookup_rtxcur", mntinfo->mik_timers[0].rtxcur);
1110         SAVE_UINT32_X(ksi, "read_srtt", mntinfo->mik_timers[1].srtt);
1111         SAVE_UINT32_X(ksi, "read_deviate", mntinfo->mik_timers[1].deviate);
1112         SAVE_UINT32_X(ksi, "read_rtxcur", mntinfo->mik_timers[1].rtxcur);
1113         SAVE_UINT32_X(ksi, "write_srtt", mntinfo->mik_timers[2].srtt);
1114         SAVE_UINT32_X(ksi, "write_deviate", mntinfo->mik_timers[2].deviate);
1115         SAVE_UINT32_X(ksi, "write_rtxcur", mntinfo->mik_timers[2].rtxcur);
1116         SAVE_UINT32(ksi, mntinfo, mik_noresponse);
1117         SAVE_UINT32(ksi, mntinfo, mik_failover);
1118         SAVE_UINT32(ksi, mntinfo, mik_remap);
1119         SAVE_STRING(ksi, mntinfo, mik_curserver);
1120 }
1121 
1122 #ifdef __sparc
1123 static void
1124 save_sfmmu_global_stat(kstat_t *kp, ks_instance_t *ksi)
1125 {
1126         struct sfmmu_global_stat *sfmmug =
1127             (struct sfmmu_global_stat *)(kp->ks_data);
1128 
1129         assert(kp->ks_data_size == sizeof (struct sfmmu_global_stat));
1130 
1131         SAVE_INT32(ksi, sfmmug, sf_tsb_exceptions);
1132         SAVE_INT32(ksi, sfmmug, sf_tsb_raise_exception);
1133         SAVE_INT32(ksi, sfmmug, sf_pagefaults);
1134         SAVE_INT32(ksi, sfmmug, sf_uhash_searches);
1135         SAVE_INT32(ksi, sfmmug, sf_uhash_links);
1136         SAVE_INT32(ksi, sfmmug, sf_khash_searches);
1137         SAVE_INT32(ksi, sfmmug, sf_khash_links);
1138         SAVE_INT32(ksi, sfmmug, sf_swapout);
1139         SAVE_INT32(ksi, sfmmug, sf_tsb_alloc);
1140         SAVE_INT32(ksi, sfmmug, sf_tsb_allocfail);
1141         SAVE_INT32(ksi, sfmmug, sf_tsb_sectsb_create);
1142         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_alloc);
1143         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_alloc);
1144         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_allocfail);
1145         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_allocfail);
1146         SAVE_INT32(ksi, sfmmug, sf_tteload8k);
1147         SAVE_INT32(ksi, sfmmug, sf_tteload64k);
1148         SAVE_INT32(ksi, sfmmug, sf_tteload512k);
1149         SAVE_INT32(ksi, sfmmug, sf_tteload4m);
1150         SAVE_INT32(ksi, sfmmug, sf_tteload32m);
1151         SAVE_INT32(ksi, sfmmug, sf_tteload256m);
1152         SAVE_INT32(ksi, sfmmug, sf_tsb_load8k);
1153         SAVE_INT32(ksi, sfmmug, sf_tsb_load4m);
1154         SAVE_INT32(ksi, sfmmug, sf_hblk_hit);
1155         SAVE_INT32(ksi, sfmmug, sf_hblk8_ncreate);
1156         SAVE_INT32(ksi, sfmmug, sf_hblk8_nalloc);
1157         SAVE_INT32(ksi, sfmmug, sf_hblk1_ncreate);
1158         SAVE_INT32(ksi, sfmmug, sf_hblk1_nalloc);
1159         SAVE_INT32(ksi, sfmmug, sf_hblk_slab_cnt);
1160         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_cnt);
1161         SAVE_INT32(ksi, sfmmug, sf_hblk_recurse_cnt);
1162         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_hit);
1163         SAVE_INT32(ksi, sfmmug, sf_get_free_success);
1164         SAVE_INT32(ksi, sfmmug, sf_get_free_throttle);
1165         SAVE_INT32(ksi, sfmmug, sf_get_free_fail);
1166         SAVE_INT32(ksi, sfmmug, sf_put_free_success);
1167         SAVE_INT32(ksi, sfmmug, sf_put_free_fail);
1168         SAVE_INT32(ksi, sfmmug, sf_pgcolor_conflict);
1169         SAVE_INT32(ksi, sfmmug, sf_uncache_conflict);
1170         SAVE_INT32(ksi, sfmmug, sf_unload_conflict);
1171         SAVE_INT32(ksi, sfmmug, sf_ism_uncache);
1172         SAVE_INT32(ksi, sfmmug, sf_ism_recache);
1173         SAVE_INT32(ksi, sfmmug, sf_recache);
1174         SAVE_INT32(ksi, sfmmug, sf_steal_count);
1175         SAVE_INT32(ksi, sfmmug, sf_pagesync);
1176         SAVE_INT32(ksi, sfmmug, sf_clrwrt);
1177         SAVE_INT32(ksi, sfmmug, sf_pagesync_invalid);
1178         SAVE_INT32(ksi, sfmmug, sf_kernel_xcalls);
1179         SAVE_INT32(ksi, sfmmug, sf_user_xcalls);
1180         SAVE_INT32(ksi, sfmmug, sf_tsb_grow);
1181         SAVE_INT32(ksi, sfmmug, sf_tsb_shrink);
1182         SAVE_INT32(ksi, sfmmug, sf_tsb_resize_failures);
1183         SAVE_INT32(ksi, sfmmug, sf_tsb_reloc);
1184         SAVE_INT32(ksi, sfmmug, sf_user_vtop);
1185         SAVE_INT32(ksi, sfmmug, sf_ctx_inv);
1186         SAVE_INT32(ksi, sfmmug, sf_tlb_reprog_pgsz);
1187         SAVE_INT32(ksi, sfmmug, sf_region_remap_demap);
1188         SAVE_INT32(ksi, sfmmug, sf_create_scd);
1189         SAVE_INT32(ksi, sfmmug, sf_join_scd);
1190         SAVE_INT32(ksi, sfmmug, sf_leave_scd);
1191         SAVE_INT32(ksi, sfmmug, sf_destroy_scd);
1192 }
1193 #endif
1194 
1195 #ifdef __sparc
1196 static void
1197 save_sfmmu_tsbsize_stat(kstat_t *kp, ks_instance_t *ksi)
1198 {
1199         struct sfmmu_tsbsize_stat *sfmmut;
1200 
1201         assert(kp->ks_data_size == sizeof (struct sfmmu_tsbsize_stat));
1202         sfmmut = (struct sfmmu_tsbsize_stat *)(kp->ks_data);
1203 
1204         SAVE_INT32(ksi, sfmmut, sf_tsbsz_8k);
1205         SAVE_INT32(ksi, sfmmut, sf_tsbsz_16k);
1206         SAVE_INT32(ksi, sfmmut, sf_tsbsz_32k);
1207         SAVE_INT32(ksi, sfmmut, sf_tsbsz_64k);
1208         SAVE_INT32(ksi, sfmmut, sf_tsbsz_128k);
1209         SAVE_INT32(ksi, sfmmut, sf_tsbsz_256k);
1210         SAVE_INT32(ksi, sfmmut, sf_tsbsz_512k);
1211         SAVE_INT32(ksi, sfmmut, sf_tsbsz_1m);
1212         SAVE_INT32(ksi, sfmmut, sf_tsbsz_2m);
1213         SAVE_INT32(ksi, sfmmut, sf_tsbsz_4m);
1214 }
1215 #endif
1216 
1217 #ifdef __sparc
1218 static void
1219 save_simmstat(kstat_t *kp, ks_instance_t *ksi)
1220 {
1221         uchar_t *simmstat;
1222         char    *simm_buf;
1223         char    *list = NULL;
1224         int     i;
1225 
1226         assert(kp->ks_data_size == sizeof (uchar_t) * SIMM_COUNT);
1227 
1228         for (i = 0, simmstat = (uchar_t *)(kp->ks_data); i < SIMM_COUNT - 1;
1229             i++, simmstat++) {
1230                 if (list == NULL) {
1231                         (void) asprintf(&simm_buf, "%d,", *simmstat);
1232                 } else {
1233                         (void) asprintf(&simm_buf, "%s%d,", list, *simmstat);
1234                         free(list);
1235                 }
1236                 list = simm_buf;
1237         }
1238 
1239         (void) asprintf(&simm_buf, "%s%d", list, *simmstat);
1240         SAVE_STRING_X(ksi, "status", simm_buf);
1241         free(list);
1242         free(simm_buf);
1243 }
1244 #endif
1245 
1246 #ifdef __sparc
1247 /*
1248  * Helper function for save_temperature().
1249  */
1250 static char *
1251 short_array_to_string(short *shortp, int len)
1252 {
1253         char    *list = NULL;
1254         char    *list_buf;
1255 
1256         for (; len > 1; len--, shortp++) {
1257                 if (list == NULL) {
1258                         (void) asprintf(&list_buf, "%hd,", *shortp);
1259                 } else {
1260                         (void) asprintf(&list_buf, "%s%hd,", list, *shortp);
1261                         free(list);
1262                 }
1263                 list = list_buf;
1264         }
1265 
1266         (void) asprintf(&list_buf, "%s%hd", list, *shortp);
1267         free(list);
1268         return (list_buf);
1269 }
1270 
1271 static void
1272 save_temperature(kstat_t *kp, ks_instance_t *ksi)
1273 {
1274         struct temp_stats *temps = (struct temp_stats *)(kp->ks_data);
1275         char    *buf;
1276 
1277         assert(kp->ks_data_size == sizeof (struct temp_stats));
1278 
1279         SAVE_UINT32(ksi, temps, index);
1280 
1281         buf = short_array_to_string(temps->l1, L1_SZ);
1282         SAVE_STRING_X(ksi, "l1", buf);
1283         free(buf);
1284 
1285         buf = short_array_to_string(temps->l2, L2_SZ);
1286         SAVE_STRING_X(ksi, "l2", buf);
1287         free(buf);
1288 
1289         buf = short_array_to_string(temps->l3, L3_SZ);
1290         SAVE_STRING_X(ksi, "l3", buf);
1291         free(buf);
1292 
1293         buf = short_array_to_string(temps->l4, L4_SZ);
1294         SAVE_STRING_X(ksi, "l4", buf);
1295         free(buf);
1296 
1297         buf = short_array_to_string(temps->l5, L5_SZ);
1298         SAVE_STRING_X(ksi, "l5", buf);
1299         free(buf);
1300 
1301         SAVE_INT32(ksi, temps, max);
1302         SAVE_INT32(ksi, temps, min);
1303         SAVE_INT32(ksi, temps, state);
1304         SAVE_INT32(ksi, temps, temp_cnt);
1305         SAVE_INT32(ksi, temps, shutdown_cnt);
1306         SAVE_INT32(ksi, temps, version);
1307         SAVE_INT32(ksi, temps, trend);
1308         SAVE_INT32(ksi, temps, override);
1309 }
1310 #endif
1311 
1312 #ifdef __sparc
1313 static void
1314 save_temp_over(kstat_t *kp, ks_instance_t *ksi)
1315 {
1316         short   *sh = (short *)(kp->ks_data);
1317         char    *value;
1318 
1319         assert(kp->ks_data_size == sizeof (short));
1320 
1321         (void) asprintf(&value, "%hu", *sh);
1322         SAVE_STRING_X(ksi, "override", value);
1323         free(value);
1324 }
1325 #endif
1326 
1327 #ifdef __sparc
1328 static void
1329 save_ps_shadow(kstat_t *kp, ks_instance_t *ksi)
1330 {
1331         uchar_t *uchar = (uchar_t *)(kp->ks_data);
1332 
1333         assert(kp->ks_data_size == SYS_PS_COUNT);
1334 
1335         SAVE_CHAR_X(ksi, "core_0", *uchar++);
1336         SAVE_CHAR_X(ksi, "core_1", *uchar++);
1337         SAVE_CHAR_X(ksi, "core_2", *uchar++);
1338         SAVE_CHAR_X(ksi, "core_3", *uchar++);
1339         SAVE_CHAR_X(ksi, "core_4", *uchar++);
1340         SAVE_CHAR_X(ksi, "core_5", *uchar++);
1341         SAVE_CHAR_X(ksi, "core_6", *uchar++);
1342         SAVE_CHAR_X(ksi, "core_7", *uchar++);
1343         SAVE_CHAR_X(ksi, "pps_0", *uchar++);
1344         SAVE_CHAR_X(ksi, "clk_33", *uchar++);
1345         SAVE_CHAR_X(ksi, "clk_50", *uchar++);
1346         SAVE_CHAR_X(ksi, "v5_p", *uchar++);
1347         SAVE_CHAR_X(ksi, "v12_p", *uchar++);
1348         SAVE_CHAR_X(ksi, "v5_aux", *uchar++);
1349         SAVE_CHAR_X(ksi, "v5_p_pch", *uchar++);
1350         SAVE_CHAR_X(ksi, "v12_p_pch", *uchar++);
1351         SAVE_CHAR_X(ksi, "v3_pch", *uchar++);
1352         SAVE_CHAR_X(ksi, "v5_pch", *uchar++);
1353         SAVE_CHAR_X(ksi, "p_fan", *uchar++);
1354 }
1355 #endif
1356 
1357 #ifdef __sparc
1358 static void
1359 save_fault_list(kstat_t *kp, ks_instance_t *ksi)
1360 {
1361         struct ft_list *fault;
1362         char    name[KSTAT_STRLEN + 7];
1363         int     i;
1364 
1365         for (i = 1, fault = (struct ft_list *)(kp->ks_data);
1366             i <= 999999 && i <= kp->ks_data_size / sizeof (struct ft_list);
1367             i++, fault++) {
1368                 (void) snprintf(name, sizeof (name), "unit_%d", i);
1369                 SAVE_INT32_X(ksi, name, fault->unit);
1370                 (void) snprintf(name, sizeof (name), "type_%d", i);
1371                 SAVE_INT32_X(ksi, name, fault->type);
1372                 (void) snprintf(name, sizeof (name), "fclass_%d", i);
1373                 SAVE_INT32_X(ksi, name, fault->fclass);
1374                 (void) snprintf(name, sizeof (name), "create_time_%d", i);
1375                 SAVE_HRTIME_X(ksi, name, fault->create_time);
1376                 (void) snprintf(name, sizeof (name), "msg_%d", i);
1377                 SAVE_STRING_X(ksi, name, fault->msg);
1378         }
1379 }
1380 #endif
1381 
1382 static void
1383 save_named(kstat_t *kp, ks_instance_t *ksi)
1384 {
1385         kstat_named_t *knp;
1386         int     n;
1387 
1388         for (n = kp->ks_ndata, knp = KSTAT_NAMED_PTR(kp); n > 0; n--, knp++) {
1389                 switch (knp->data_type) {
1390                 case KSTAT_DATA_CHAR:
1391                         nvpair_insert(ksi, knp->name,
1392                             (ks_value_t *)&knp->value, KSTAT_DATA_CHAR);
1393                         break;
1394                 case KSTAT_DATA_INT32:
1395                         nvpair_insert(ksi, knp->name,
1396                             (ks_value_t *)&knp->value, KSTAT_DATA_INT32);
1397                         break;
1398                 case KSTAT_DATA_UINT32:
1399                         nvpair_insert(ksi, knp->name,
1400                             (ks_value_t *)&knp->value, KSTAT_DATA_UINT32);
1401                         break;
1402                 case KSTAT_DATA_INT64:
1403                         nvpair_insert(ksi, knp->name,
1404                             (ks_value_t *)&knp->value, KSTAT_DATA_INT64);
1405                         break;
1406                 case KSTAT_DATA_UINT64:
1407                         nvpair_insert(ksi, knp->name,
1408                             (ks_value_t *)&knp->value, KSTAT_DATA_UINT64);
1409                         break;
1410                 case KSTAT_DATA_STRING:
1411                         SAVE_STRING_X(ksi, knp->name, KSTAT_NAMED_STR_PTR(knp));
1412                         break;
1413                 default:
1414                         assert(B_FALSE); /* Invalid data type */
1415                         break;
1416                 }
1417         }
1418 }
1419 
1420 static void
1421 save_intr(kstat_t *kp, ks_instance_t *ksi)
1422 {
1423         kstat_intr_t *intr = KSTAT_INTR_PTR(kp);
1424         char    *intr_names[] = {"hard", "soft", "watchdog", "spurious",
1425             "multiple_service"};
1426         int     n;
1427 
1428         for (n = 0; n < KSTAT_NUM_INTRS; n++)
1429                 SAVE_UINT32_X(ksi, intr_names[n], intr->intrs[n]);
1430 }
1431 
1432 static void
1433 save_io(kstat_t *kp, ks_instance_t *ksi)
1434 {
1435         kstat_io_t      *ksio = KSTAT_IO_PTR(kp);
1436 
1437         SAVE_UINT64(ksi, ksio, nread);
1438         SAVE_UINT64(ksi, ksio, nwritten);
1439         SAVE_UINT32(ksi, ksio, reads);
1440         SAVE_UINT32(ksi, ksio, writes);
1441         SAVE_HRTIME(ksi, ksio, wtime);
1442         SAVE_HRTIME(ksi, ksio, wlentime);
1443         SAVE_HRTIME(ksi, ksio, wlastupdate);
1444         SAVE_HRTIME(ksi, ksio, rtime);
1445         SAVE_HRTIME(ksi, ksio, rlentime);
1446         SAVE_HRTIME(ksi, ksio, rlastupdate);
1447         SAVE_UINT32(ksi, ksio, wcnt);
1448         SAVE_UINT32(ksi, ksio, rcnt);
1449 }
1450 
1451 static void
1452 save_timer(kstat_t *kp, ks_instance_t *ksi)
1453 {
1454         kstat_timer_t   *ktimer = KSTAT_TIMER_PTR(kp);
1455 
1456         SAVE_STRING(ksi, ktimer, name);
1457         SAVE_UINT64(ksi, ktimer, num_events);
1458         SAVE_HRTIME(ksi, ktimer, elapsed_time);
1459         SAVE_HRTIME(ksi, ktimer, min_time);
1460         SAVE_HRTIME(ksi, ktimer, max_time);
1461         SAVE_HRTIME(ksi, ktimer, start_time);
1462         SAVE_HRTIME(ksi, ktimer, stop_time);
1463 }