1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013 David Hoeppner. All rights reserved.
  25  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  26  */
  27 
  28 /*
  29  * Display kernel statistics
  30  *
  31  * This is a reimplementation of the perl kstat command originally found
  32  * under usr/src/cmd/kstat/kstat.pl
  33  *
  34  * Incompatibilities:
  35  *      - perl regular expressions replaced with extended REs bracketed by '/'
  36  *
  37  * Flags added:
  38  *      -C      similar to the -p option but value is separated by a colon
  39  *      -h      display help
  40  *      -j      json format
  41  */
  42 
  43 #include <assert.h>
  44 #include <ctype.h>
  45 #include <errno.h>
  46 #include <kstat.h>
  47 #include <langinfo.h>
  48 #include <libgen.h>
  49 #include <limits.h>
  50 #include <locale.h>
  51 #include <signal.h>
  52 #include <stddef.h>
  53 #include <stdio.h>
  54 #include <stdlib.h>
  55 #include <string.h>
  56 #include <strings.h>
  57 #include <time.h>
  58 #include <unistd.h>
  59 #include <sys/list.h>
  60 #include <sys/time.h>
  61 #include <sys/types.h>
  62 
  63 #include "kstat.h"
  64 #include "statcommon.h"
  65 
  66 char    *cmdname = "kstat";     /* Name of this command */
  67 int     caught_cont = 0;        /* Have caught a SIGCONT */
  68 
  69 static uint_t   g_timestamp_fmt = NODATE;
  70 
  71 /* Helper flag - header was printed already? */
  72 static boolean_t g_headerflg;
  73 
  74 /* Saved command line options */
  75 static boolean_t g_cflg = B_FALSE;
  76 static boolean_t g_jflg = B_FALSE;
  77 static boolean_t g_lflg = B_FALSE;
  78 static boolean_t g_pflg = B_FALSE;
  79 static boolean_t g_qflg = B_FALSE;
  80 static ks_pattern_t     g_ks_class = {"*", 0};
  81 
  82 /* Return zero if a selector did match */
  83 static int      g_matched = 1;
  84 
  85 /* Sorted list of kstat instances */
  86 static list_t   instances_list;
  87 static list_t   selector_list;
  88 
  89 int
  90 main(int argc, char **argv)
  91 {
  92         ks_selector_t   *nselector;
  93         ks_selector_t   *uselector;
  94         kstat_ctl_t     *kc;
  95         hrtime_t        start_n;
  96         hrtime_t        period_n;
  97         boolean_t       errflg = B_FALSE;
  98         boolean_t       nselflg = B_FALSE;
  99         boolean_t       uselflg = B_FALSE;
 100         char            *q;
 101         int             count = 1;
 102         int             infinite_cycles = 0;
 103         int             interval = 0;
 104         int             n = 0;
 105         int             c, m, tmp;
 106 
 107         (void) setlocale(LC_ALL, "");
 108 #if !defined(TEXT_DOMAIN)               /* Should be defined by cc -D */
 109 #define TEXT_DOMAIN "SYS_TEST"          /* Use this only if it wasn't */
 110 #endif
 111         (void) textdomain(TEXT_DOMAIN);
 112 
 113         /*
 114          * Create the selector list and a dummy default selector to match
 115          * everything. While we process the cmdline options we will add
 116          * selectors to this list.
 117          */
 118         list_create(&selector_list, sizeof (ks_selector_t),
 119             offsetof(ks_selector_t, ks_next));
 120 
 121         nselector = new_selector();
 122 
 123         /*
 124          * Parse named command line arguments.
 125          */
 126         while ((c = getopt(argc, argv, "h?CqjlpT:m:i:n:s:c:")) != EOF)
 127                 switch (c) {
 128                 case 'h':
 129                 case '?':
 130                         usage();
 131                         exit(0);
 132                         break;
 133                 case 'C':
 134                         g_pflg = g_cflg = B_TRUE;
 135                         break;
 136                 case 'q':
 137                         g_qflg = B_TRUE;
 138                         break;
 139                 case 'j':
 140                         g_jflg = B_TRUE;
 141                         break;
 142                 case 'l':
 143                         g_pflg = g_lflg = B_TRUE;
 144                         break;
 145                 case 'p':
 146                         g_pflg = B_TRUE;
 147                         break;
 148                 case 'T':
 149                         switch (*optarg) {
 150                         case 'd':
 151                                 g_timestamp_fmt = DDATE;
 152                                 break;
 153                         case 'u':
 154                                 g_timestamp_fmt = UDATE;
 155                                 break;
 156                         default:
 157                                 errflg = B_TRUE;
 158                         }
 159                         break;
 160                 case 'm':
 161                         nselflg = B_TRUE;
 162                         nselector->ks_module.pstr =
 163                             (char *)ks_safe_strdup(optarg);
 164                         break;
 165                 case 'i':
 166                         nselflg = B_TRUE;
 167                         nselector->ks_instance.pstr =
 168                             (char *)ks_safe_strdup(optarg);
 169                         break;
 170                 case 'n':
 171                         nselflg = B_TRUE;
 172                         nselector->ks_name.pstr =
 173                             (char *)ks_safe_strdup(optarg);
 174                         break;
 175                 case 's':
 176                         nselflg = B_TRUE;
 177                         nselector->ks_statistic.pstr =
 178                             (char *)ks_safe_strdup(optarg);
 179                         break;
 180                 case 'c':
 181                         g_ks_class.pstr =
 182                             (char *)ks_safe_strdup(optarg);
 183                         break;
 184                 default:
 185                         errflg = B_TRUE;
 186                         break;
 187                 }
 188 
 189         if (g_qflg && (g_jflg || g_pflg)) {
 190                 (void) fprintf(stderr, gettext(
 191                     "-q and -lpj are mutually exclusive\n"));
 192                 errflg = B_TRUE;
 193         }
 194 
 195         if (errflg) {
 196                 usage();
 197                 exit(2);
 198         }
 199 
 200         argc -= optind;
 201         argv += optind;
 202 
 203         /*
 204          * Consume the rest of the command line. Parsing the
 205          * unnamed command line arguments.
 206          */
 207         while (argc--) {
 208                 errno = 0;
 209                 tmp = strtoul(*argv, &q, 10);
 210                 if (tmp == ULONG_MAX && errno == ERANGE) {
 211                         if (n == 0) {
 212                                 (void) fprintf(stderr, gettext(
 213                                     "Interval is too large\n"));
 214                         } else if (n == 1) {
 215                                 (void) fprintf(stderr, gettext(
 216                                     "Count is too large\n"));
 217                         }
 218                         usage();
 219                         exit(2);
 220                 }
 221 
 222                 if (errno != 0 || *q != '\0') {
 223                         m = 0;
 224                         uselector = new_selector();
 225                         while ((q = (char *)strsep(argv, ":")) != NULL) {
 226                                 m++;
 227                                 if (m > 4) {
 228                                         free(uselector);
 229                                         usage();
 230                                         exit(2);
 231                                 }
 232 
 233                                 if (*q != '\0') {
 234                                         switch (m) {
 235                                         case 1:
 236                                                 uselector->ks_module.pstr =
 237                                                     (char *)ks_safe_strdup(q);
 238                                                 break;
 239                                         case 2:
 240                                                 uselector->ks_instance.pstr =
 241                                                     (char *)ks_safe_strdup(q);
 242                                                 break;
 243                                         case 3:
 244                                                 uselector->ks_name.pstr =
 245                                                     (char *)ks_safe_strdup(q);
 246                                                 break;
 247                                         case 4:
 248                                                 uselector->ks_statistic.pstr =
 249                                                     (char *)ks_safe_strdup(q);
 250                                                 break;
 251                                         default:
 252                                                 assert(B_FALSE);
 253                                         }
 254                                 }
 255                         }
 256 
 257                         uselflg = B_TRUE;
 258                         list_insert_tail(&selector_list, uselector);
 259                 } else {
 260                         if (tmp < 1) {
 261                                 if (n == 0) {
 262                                         (void) fprintf(stderr, gettext(
 263                                             "Interval must be an "
 264                                             "integer >= 1"));
 265                                 } else if (n == 1) {
 266                                         (void) fprintf(stderr, gettext(
 267                                             "Count must be an integer >= 1"));
 268                                 }
 269                                 usage();
 270                                 exit(2);
 271                         } else {
 272                                 if (n == 0) {
 273                                         interval = tmp;
 274                                         count = -1;
 275                                 } else if (n == 1) {
 276                                         count = tmp;
 277                                 } else {
 278                                         usage();
 279                                         exit(2);
 280                                 }
 281                         }
 282                         n++;
 283                 }
 284                 argv++;
 285         }
 286 
 287         /*
 288          * Check if we founded a named selector on the cmdline.
 289          */
 290         if (uselflg) {
 291                 if (nselflg) {
 292                         (void) fprintf(stderr, gettext(
 293                             "[module[:instance[:name[:statistic]]]] and "
 294                             "-m -i -n -s are mutually exclusive"));
 295                         usage();
 296                         exit(2);
 297                 } else {
 298                         free(nselector);
 299                 }
 300         } else {
 301                 list_insert_tail(&selector_list, nselector);
 302         }
 303 
 304         assert(!list_is_empty(&selector_list));
 305 
 306         list_create(&instances_list, sizeof (ks_instance_t),
 307             offsetof(ks_instance_t, ks_next));
 308 
 309         while ((kc = kstat_open()) == NULL) {
 310                 if (errno == EAGAIN) {
 311                         (void) poll(NULL, 0, 200);
 312                 } else {
 313                         perror("kstat_open");
 314                         exit(3);
 315                 }
 316         }
 317 
 318         if (count > 1) {
 319                 if (signal(SIGCONT, cont_handler) == SIG_ERR) {
 320                         (void) fprintf(stderr, gettext(
 321                             "signal failed"));
 322                         exit(3);
 323                 }
 324         }
 325 
 326         period_n = (hrtime_t)interval * NANOSEC;
 327         start_n = gethrtime();
 328 
 329         while (count == -1 || count-- > 0) {
 330                 ks_instances_read(kc);
 331                 ks_instances_print();
 332 
 333                 if (interval && count) {
 334                         ks_sleep_until(&start_n, period_n, infinite_cycles,
 335                             &caught_cont);
 336                         (void) kstat_chain_update(kc);
 337                         (void) putchar('\n');
 338                 }
 339         }
 340 
 341         (void) kstat_close(kc);
 342 
 343         return (g_matched);
 344 }
 345 
 346 /*
 347  * Print usage.
 348  */
 349 static void
 350 usage(void)
 351 {
 352         (void) fprintf(stderr, gettext(
 353             "Usage:\n"
 354             "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
 355             "      [ -m module ] [ -i instance ] [ -n name ] [ -s statistic ]\n"
 356             "      [ interval [ count ] ]\n"
 357             "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
 358             "      [ module[:instance[:name[:statistic]]] ... ]\n"
 359             "      [ interval [ count ] ]\n"));
 360 }
 361 
 362 /*
 363  * Sort compare function.
 364  */
 365 static int
 366 compare_instances(ks_instance_t *l_arg, ks_instance_t *r_arg)
 367 {
 368         int     rval;
 369 
 370         rval = strcasecmp(l_arg->ks_module, r_arg->ks_module);
 371         if (rval == 0) {
 372                 if (l_arg->ks_instance == r_arg->ks_instance) {
 373                         return (strcasecmp(l_arg->ks_name, r_arg->ks_name));
 374                 } else if (l_arg->ks_instance < r_arg->ks_instance) {
 375                         return (-1);
 376                 } else {
 377                         return (1);
 378                 }
 379         } else {
 380                 return (rval);
 381         }
 382 }
 383 
 384 static char *
 385 ks_safe_strdup(char *str)
 386 {
 387         char    *ret;
 388 
 389         if (str == NULL) {
 390                 return (NULL);
 391         }
 392 
 393         while ((ret = strdup(str)) == NULL) {
 394                 if (errno == EAGAIN) {
 395                         (void) poll(NULL, 0, 200);
 396                 } else {
 397                         perror("strdup");
 398                         exit(3);
 399                 }
 400         }
 401 
 402         return (ret);
 403 }
 404 
 405 static void
 406 ks_sleep_until(hrtime_t *wakeup, hrtime_t interval, int forever,
 407     int *caught_cont)
 408 {
 409         hrtime_t        now, pause, pause_left;
 410         struct timespec pause_tv;
 411         int             status;
 412 
 413         now = gethrtime();
 414         pause = *wakeup + interval - now;
 415 
 416         if (pause <= 0 || pause < (interval / 4)) {
 417                 if (forever || *caught_cont) {
 418                         *wakeup = now + interval;
 419                         pause = interval;
 420                 } else {
 421                         pause = interval / 2;
 422                         *wakeup += interval;
 423                 }
 424         } else {
 425                 *wakeup += interval;
 426         }
 427 
 428         if (pause < 1000) {
 429                 return;
 430         }
 431 
 432         pause_left = pause;
 433         do {
 434                 pause_tv.tv_sec = pause_left / NANOSEC;
 435                 pause_tv.tv_nsec = pause_left % NANOSEC;
 436                 status = nanosleep(&pause_tv, (struct timespec *)NULL);
 437                 if (status < 0) {
 438                         if (errno == EINTR) {
 439                                 now = gethrtime();
 440                                 pause_left = *wakeup - now;
 441                                 if (pause_left < 1000) {
 442                                         return;
 443                                 }
 444                         } else {
 445                                 perror("nanosleep");
 446                                 exit(3);
 447                         }
 448                 }
 449         } while (status != 0);
 450 }
 451 
 452 /*
 453  * Inserts an instance in the per selector list.
 454  */
 455 static void
 456 nvpair_insert(ks_instance_t *ksi, char *name, ks_value_t *value,
 457     uchar_t data_type)
 458 {
 459         ks_nvpair_t     *instance;
 460         ks_nvpair_t     *tmp;
 461 
 462         instance = (ks_nvpair_t *)malloc(sizeof (ks_nvpair_t));
 463         if (instance == NULL) {
 464                 perror("malloc");
 465                 exit(3);
 466         }
 467 
 468         (void) strlcpy(instance->name, name, KSTAT_STRLEN);
 469         (void) memcpy(&instance->value, value, sizeof (ks_value_t));
 470         instance->data_type = data_type;
 471 
 472         tmp = list_head(&ksi->ks_nvlist);
 473         while (tmp != NULL && strcasecmp(instance->name, tmp->name) > 0)
 474                 tmp = list_next(&ksi->ks_nvlist, tmp);
 475 
 476         list_insert_before(&ksi->ks_nvlist, tmp, instance);
 477 }
 478 
 479 /*
 480  * Allocates a new all-matching selector.
 481  */
 482 static ks_selector_t *
 483 new_selector(void)
 484 {
 485         ks_selector_t   *selector;
 486 
 487         selector = (ks_selector_t *)malloc(sizeof (ks_selector_t));
 488         if (selector == NULL) {
 489                 perror("malloc");
 490                 exit(3);
 491         }
 492 
 493         list_link_init(&selector->ks_next);
 494 
 495         selector->ks_module.pstr = "*";
 496         selector->ks_instance.pstr = "*";
 497         selector->ks_name.pstr = "*";
 498         selector->ks_statistic.pstr = "*";
 499 
 500         return (selector);
 501 }
 502 
 503 /*
 504  * This function was taken from the perl kstat module code - please
 505  * see for further comments there.
 506  */
 507 static kstat_raw_reader_t
 508 lookup_raw_kstat_fn(char *module, char *name)
 509 {
 510         char            key[KSTAT_STRLEN * 2];
 511         register char   *f, *t;
 512         int             n = 0;
 513 
 514         for (f = module, t = key; *f != '\0'; f++, t++) {
 515                 while (*f != '\0' && isdigit(*f))
 516                         f++;
 517                 *t = *f;
 518         }
 519         *t++ = ':';
 520 
 521         for (f = name; *f != '\0'; f++, t++) {
 522                 while (*f != '\0' && isdigit(*f))
 523                         f++;
 524                 *t = *f;
 525         }
 526         *t = '\0';
 527 
 528         while (ks_raw_lookup[n].fn != NULL) {
 529                 if (strncmp(ks_raw_lookup[n].name, key, strlen(key)) == 0)
 530                         return (ks_raw_lookup[n].fn);
 531                 n++;
 532         }
 533 
 534         return (0);
 535 }
 536 
 537 /*
 538  * Match a string against a shell glob or extended regular expression.
 539  */
 540 static boolean_t
 541 ks_match(const char *str, ks_pattern_t *pattern)
 542 {
 543         int     regcode;
 544         char    *regstr;
 545         char    *errbuf;
 546         size_t  bufsz;
 547 
 548         if (pattern->pstr != NULL && gmatch(pattern->pstr, "/*/") != 0) {
 549                 /* All regex patterns are strdup'd copies */
 550                 regstr = pattern->pstr + 1;
 551                 *(strrchr(regstr, '/')) = '\0';
 552 
 553                 regcode = regcomp(&pattern->preg, regstr,
 554                     REG_EXTENDED | REG_NOSUB);
 555                 if (regcode != 0) {
 556                         bufsz = regerror(regcode, NULL, NULL, 0);
 557                         if (bufsz != 0) {
 558                                 errbuf = malloc(bufsz);
 559                                 if (errbuf == NULL) {
 560                                         perror("malloc");
 561                                         exit(3);
 562                                 }
 563                                 (void) regerror(regcode, NULL, errbuf, bufsz);
 564                                 (void) fprintf(stderr, "kstat: %s\n", errbuf);
 565                         }
 566                         usage();
 567                         exit(2);
 568                 }
 569 
 570                 pattern->pstr = NULL;
 571         }
 572 
 573         if (pattern->pstr == NULL) {
 574                 return (regexec(&pattern->preg, str, 0, NULL, 0) == 0);
 575         }
 576 
 577         return ((gmatch(str, pattern->pstr) != 0));
 578 }
 579 
 580 /*
 581  * Iterate over all kernel statistics and save matches.
 582  */
 583 static void
 584 ks_instances_read(kstat_ctl_t *kc)
 585 {
 586         kstat_raw_reader_t save_raw = NULL;
 587         kid_t           id;
 588         ks_selector_t   *selector;
 589         ks_instance_t   *ksi;
 590         ks_instance_t   *tmp;
 591         kstat_t         *kp;
 592         boolean_t       skip;
 593 
 594         for (kp = kc->kc_chain; kp != NULL; kp = kp->ks_next) {
 595                 /* Don't bother storing the kstat headers */
 596                 if (strncmp(kp->ks_name, "kstat_", 6) == 0) {
 597                         continue;
 598                 }
 599 
 600                 /* Don't bother storing raw stats we don't understand */
 601                 if (kp->ks_type == KSTAT_TYPE_RAW) {
 602                         save_raw = lookup_raw_kstat_fn(kp->ks_module,
 603                             kp->ks_name);
 604                         if (save_raw == NULL) {
 605 #ifdef REPORT_UNKNOWN
 606                                 (void) fprintf(stderr,
 607                                     "Unknown kstat type %s:%d:%s - "
 608                                     "%d of size %d\n", kp->ks_module,
 609                                     kp->ks_instance, kp->ks_name,
 610                                     kp->ks_ndata, kp->ks_data_size);
 611 #endif
 612                                 continue;
 613                         }
 614                 }
 615 
 616                 /*
 617                  * Iterate over the list of selectors and skip
 618                  * instances we dont want. We filter for statistics
 619                  * later, as we dont know them yet.
 620                  */
 621                 skip = B_TRUE;
 622                 selector = list_head(&selector_list);
 623                 while (selector != NULL) {
 624                         if (ks_match(kp->ks_module, &selector->ks_module) ||
 625                             ks_match(kp->ks_name, &selector->ks_name)) {
 626                                 skip = B_FALSE;
 627                                 break;
 628                         }
 629                         selector = list_next(&selector_list, selector);
 630                 }
 631 
 632                 if (skip) {
 633                         continue;
 634                 }
 635 
 636                 /*
 637                  * Allocate a new instance and fill in the values
 638                  * we know so far.
 639                  */
 640                 ksi = (ks_instance_t *)malloc(sizeof (ks_instance_t));
 641                 if (ksi == NULL) {
 642                         perror("malloc");
 643                         exit(3);
 644                 }
 645 
 646                 list_link_init(&ksi->ks_next);
 647 
 648                 (void) strlcpy(ksi->ks_module, kp->ks_module, KSTAT_STRLEN);
 649                 (void) strlcpy(ksi->ks_name, kp->ks_name, KSTAT_STRLEN);
 650                 (void) strlcpy(ksi->ks_class, kp->ks_class, KSTAT_STRLEN);
 651 
 652                 ksi->ks_instance = kp->ks_instance;
 653                 ksi->ks_snaptime = kp->ks_snaptime;
 654                 ksi->ks_type = kp->ks_type;
 655 
 656                 list_create(&ksi->ks_nvlist, sizeof (ks_nvpair_t),
 657                     offsetof(ks_nvpair_t, nv_next));
 658 
 659                 SAVE_HRTIME_X(ksi, "crtime", kp->ks_crtime);
 660                 SAVE_HRTIME_X(ksi, "snaptime", kp->ks_snaptime);
 661                 if (g_pflg) {
 662                         SAVE_STRING_X(ksi, "class", kp->ks_class);
 663                 }
 664 
 665                 /* Insert this instance into a sorted list */
 666                 tmp = list_head(&instances_list);
 667                 while (tmp != NULL && compare_instances(ksi, tmp) > 0)
 668                         tmp = list_next(&instances_list, tmp);
 669 
 670                 list_insert_before(&instances_list, tmp, ksi);
 671 
 672                 /* Read the actual statistics */
 673                 id = kstat_read(kc, kp, NULL);
 674                 if (id == -1) {
 675 #ifdef REPORT_UNKNOWN
 676                         perror("kstat_read");
 677 #endif
 678                         continue;
 679                 }
 680 
 681                 switch (kp->ks_type) {
 682                 case KSTAT_TYPE_RAW:
 683                         save_raw(kp, ksi);
 684                         break;
 685                 case KSTAT_TYPE_NAMED:
 686                         save_named(kp, ksi);
 687                         break;
 688                 case KSTAT_TYPE_INTR:
 689                         save_intr(kp, ksi);
 690                         break;
 691                 case KSTAT_TYPE_IO:
 692                         save_io(kp, ksi);
 693                         break;
 694                 case KSTAT_TYPE_TIMER:
 695                         save_timer(kp, ksi);
 696                         break;
 697                 default:
 698                         assert(B_FALSE); /* Invalid type */
 699                         break;
 700                 }
 701         }
 702 }
 703 
 704 /*
 705  * Print the value of a name-value pair.
 706  */
 707 static void
 708 ks_value_print(ks_nvpair_t *nvpair)
 709 {
 710         switch (nvpair->data_type) {
 711         case KSTAT_DATA_CHAR:
 712                 (void) fprintf(stdout, "%s", nvpair->value.c);
 713                 break;
 714         case KSTAT_DATA_INT32:
 715                 (void) fprintf(stdout, "%d", nvpair->value.i32);
 716                 break;
 717         case KSTAT_DATA_UINT32:
 718                 (void) fprintf(stdout, "%u", nvpair->value.ui32);
 719                 break;
 720         case KSTAT_DATA_INT64:
 721                 (void) fprintf(stdout, "%lld", nvpair->value.i64);
 722                 break;
 723         case KSTAT_DATA_UINT64:
 724                 (void) fprintf(stdout, "%llu", nvpair->value.ui64);
 725                 break;
 726         case KSTAT_DATA_STRING:
 727                 (void) fprintf(stdout, "%s", KSTAT_NAMED_STR_PTR(nvpair));
 728                 break;
 729         case KSTAT_DATA_HRTIME:
 730                 if (nvpair->value.ui64 == 0)
 731                         (void) fprintf(stdout, "0");
 732                 else
 733                         (void) fprintf(stdout, "%.9f",
 734                             nvpair->value.ui64 / 1000000000.0);
 735                 break;
 736         default:
 737                 assert(B_FALSE);
 738         }
 739 }
 740 
 741 /*
 742  * Print a single instance.
 743  */
 744 static void
 745 ks_instance_print(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 746 {
 747         if (g_headerflg) {
 748                 if (!g_pflg) {
 749                         (void) fprintf(stdout, DFLT_FMT,
 750                             ksi->ks_module, ksi->ks_instance,
 751                             ksi->ks_name, ksi->ks_class);
 752                 }
 753                 g_headerflg = B_FALSE;
 754         }
 755 
 756         if (g_pflg) {
 757                 (void) fprintf(stdout, KS_PFMT,
 758                     ksi->ks_module, ksi->ks_instance,
 759                     ksi->ks_name, nvpair->name);
 760                 if (!g_lflg) {
 761                         (void) putchar(g_cflg ? ':': '\t');
 762                         ks_value_print(nvpair);
 763                 }
 764         } else {
 765                 (void) fprintf(stdout, KS_DFMT, nvpair->name);
 766                 ks_value_print(nvpair);
 767         }
 768 
 769         (void) putchar('\n');
 770 }
 771 
 772 /*
 773  * Print a single instance in JSON format.
 774  */
 775 static void
 776 ks_instance_print_json(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 777 {
 778         if (g_headerflg) {
 779                 (void) fprintf(stdout, JSON_FMT,
 780                     ksi->ks_module, ksi->ks_instance,
 781                     ksi->ks_name, ksi->ks_class,
 782                     ksi->ks_type);
 783 
 784                 if (ksi->ks_snaptime == 0)
 785                         (void) fprintf(stdout, "\t\"snaptime\": 0,\n");
 786                 else
 787                         (void) fprintf(stdout, "\t\"snaptime\": %.9f,\n",
 788                             ksi->ks_snaptime / 1000000000.0);
 789 
 790                 (void) fprintf(stdout, "\t\"data\": {\n");
 791 
 792                 g_headerflg = B_FALSE;
 793         }
 794 
 795         (void) fprintf(stdout, KS_JFMT, nvpair->name);
 796         if (nvpair->data_type == KSTAT_DATA_STRING) {
 797                 (void) putchar('\"');
 798                 ks_value_print(nvpair);
 799                 (void) putchar('\"');
 800         } else {
 801                 ks_value_print(nvpair);
 802         }
 803         if (nvpair != list_tail(&ksi->ks_nvlist))
 804                 (void) putchar(',');
 805 
 806         (void) putchar('\n');
 807 }
 808 
 809 /*
 810  * Print all instances.
 811  */
 812 static void
 813 ks_instances_print(void)
 814 {
 815         ks_selector_t   *selector;
 816         ks_instance_t   *ksi, *ktmp;
 817         ks_nvpair_t     *nvpair, *ntmp;
 818         void            (*ks_print_fn)(ks_instance_t *, ks_nvpair_t *);
 819         char            *ks_number;
 820 
 821         if (g_timestamp_fmt != NODATE)
 822                 print_timestamp(g_timestamp_fmt);
 823 
 824         if (g_jflg) {
 825                 ks_print_fn = &ks_instance_print_json;
 826                 (void) putchar('[');
 827         } else {
 828                 ks_print_fn = &ks_instance_print;
 829         }
 830 
 831         /* Iterate over each selector */
 832         selector = list_head(&selector_list);
 833         while (selector != NULL) {
 834 
 835                 /* Iterate over each instance */
 836                 for (ksi = list_head(&instances_list); ksi != NULL;
 837                     ksi = list_next(&instances_list, ksi)) {
 838 
 839                         (void) asprintf(&ks_number, "%d", ksi->ks_instance);
 840                         if (!(ks_match(ksi->ks_module, &selector->ks_module) &&
 841                             ks_match(ksi->ks_name, &selector->ks_name) &&
 842                             ks_match(ks_number, &selector->ks_instance) &&
 843                             ks_match(ksi->ks_class, &g_ks_class))) {
 844                                 free(ks_number);
 845                                 continue;
 846                         }
 847 
 848                         free(ks_number);
 849 
 850                         /* Finally iterate over each statistic */
 851                         g_headerflg = B_TRUE;
 852                         for (nvpair = list_head(&ksi->ks_nvlist);
 853                             nvpair != NULL;
 854                             nvpair = list_next(&ksi->ks_nvlist, nvpair)) {
 855                                 if (!ks_match(nvpair->name,
 856                                     &selector->ks_statistic))
 857                                         continue;
 858 
 859                                 g_matched = 0;
 860                                 if (!g_qflg)
 861                                         (*ks_print_fn)(ksi, nvpair);
 862                         }
 863 
 864                         if (!g_headerflg) {
 865                                 if (g_jflg) {
 866                                         (void) fprintf(stdout, "\t}\n}");
 867                                         if (ksi != list_tail(&instances_list))
 868                                                 (void) putchar(',');
 869                                 } else if (!g_pflg) {
 870                                         (void) putchar('\n');
 871                                 }
 872                         }
 873                 }
 874 
 875                 selector = list_next(&selector_list, selector);
 876         }
 877 
 878         if (g_jflg)
 879                 (void) fprintf(stdout, "]\n");
 880 
 881         (void) fflush(stdout);
 882 
 883         /* Free the instances list */
 884         ksi = list_head(&instances_list);
 885         while (ksi != NULL) {
 886                 nvpair = list_head(&ksi->ks_nvlist);
 887                 while (nvpair != NULL) {
 888                         ntmp = nvpair;
 889                         nvpair = list_next(&ksi->ks_nvlist, nvpair);
 890                         list_remove(&ksi->ks_nvlist, ntmp);
 891                         if (ntmp->data_type == KSTAT_DATA_STRING)
 892                                 free(ntmp->value.str.addr.ptr);
 893                         free(ntmp);
 894                 }
 895 
 896                 ktmp = ksi;
 897                 ksi = list_next(&instances_list, ksi);
 898                 list_remove(&instances_list, ktmp);
 899                 list_destroy(&ktmp->ks_nvlist);
 900                 free(ktmp);
 901         }
 902 }
 903 
 904 static void
 905 save_cpu_stat(kstat_t *kp, ks_instance_t *ksi)
 906 {
 907         cpu_stat_t      *stat;
 908         cpu_sysinfo_t   *sysinfo;
 909         cpu_syswait_t   *syswait;
 910         cpu_vminfo_t    *vminfo;
 911 
 912         stat = (cpu_stat_t *)(kp->ks_data);
 913         sysinfo = &stat->cpu_sysinfo;
 914         syswait = &stat->cpu_syswait;
 915         vminfo  = &stat->cpu_vminfo;
 916 
 917         SAVE_UINT32_X(ksi, "idle", sysinfo->cpu[CPU_IDLE]);
 918         SAVE_UINT32_X(ksi, "user", sysinfo->cpu[CPU_USER]);
 919         SAVE_UINT32_X(ksi, "kernel", sysinfo->cpu[CPU_KERNEL]);
 920         SAVE_UINT32_X(ksi, "wait", sysinfo->cpu[CPU_WAIT]);
 921         SAVE_UINT32_X(ksi, "wait_io", sysinfo->wait[W_IO]);
 922         SAVE_UINT32_X(ksi, "wait_swap", sysinfo->wait[W_SWAP]);
 923         SAVE_UINT32_X(ksi, "wait_pio", sysinfo->wait[W_PIO]);
 924         SAVE_UINT32(ksi, sysinfo, bread);
 925         SAVE_UINT32(ksi, sysinfo, bwrite);
 926         SAVE_UINT32(ksi, sysinfo, lread);
 927         SAVE_UINT32(ksi, sysinfo, lwrite);
 928         SAVE_UINT32(ksi, sysinfo, phread);
 929         SAVE_UINT32(ksi, sysinfo, phwrite);
 930         SAVE_UINT32(ksi, sysinfo, pswitch);
 931         SAVE_UINT32(ksi, sysinfo, trap);
 932         SAVE_UINT32(ksi, sysinfo, intr);
 933         SAVE_UINT32(ksi, sysinfo, syscall);
 934         SAVE_UINT32(ksi, sysinfo, sysread);
 935         SAVE_UINT32(ksi, sysinfo, syswrite);
 936         SAVE_UINT32(ksi, sysinfo, sysfork);
 937         SAVE_UINT32(ksi, sysinfo, sysvfork);
 938         SAVE_UINT32(ksi, sysinfo, sysexec);
 939         SAVE_UINT32(ksi, sysinfo, readch);
 940         SAVE_UINT32(ksi, sysinfo, writech);
 941         SAVE_UINT32(ksi, sysinfo, rcvint);
 942         SAVE_UINT32(ksi, sysinfo, xmtint);
 943         SAVE_UINT32(ksi, sysinfo, mdmint);
 944         SAVE_UINT32(ksi, sysinfo, rawch);
 945         SAVE_UINT32(ksi, sysinfo, canch);
 946         SAVE_UINT32(ksi, sysinfo, outch);
 947         SAVE_UINT32(ksi, sysinfo, msg);
 948         SAVE_UINT32(ksi, sysinfo, sema);
 949         SAVE_UINT32(ksi, sysinfo, namei);
 950         SAVE_UINT32(ksi, sysinfo, ufsiget);
 951         SAVE_UINT32(ksi, sysinfo, ufsdirblk);
 952         SAVE_UINT32(ksi, sysinfo, ufsipage);
 953         SAVE_UINT32(ksi, sysinfo, ufsinopage);
 954         SAVE_UINT32(ksi, sysinfo, inodeovf);
 955         SAVE_UINT32(ksi, sysinfo, fileovf);
 956         SAVE_UINT32(ksi, sysinfo, procovf);
 957         SAVE_UINT32(ksi, sysinfo, intrthread);
 958         SAVE_UINT32(ksi, sysinfo, intrblk);
 959         SAVE_UINT32(ksi, sysinfo, idlethread);
 960         SAVE_UINT32(ksi, sysinfo, inv_swtch);
 961         SAVE_UINT32(ksi, sysinfo, nthreads);
 962         SAVE_UINT32(ksi, sysinfo, cpumigrate);
 963         SAVE_UINT32(ksi, sysinfo, xcalls);
 964         SAVE_UINT32(ksi, sysinfo, mutex_adenters);
 965         SAVE_UINT32(ksi, sysinfo, rw_rdfails);
 966         SAVE_UINT32(ksi, sysinfo, rw_wrfails);
 967         SAVE_UINT32(ksi, sysinfo, modload);
 968         SAVE_UINT32(ksi, sysinfo, modunload);
 969         SAVE_UINT32(ksi, sysinfo, bawrite);
 970 #ifdef  STATISTICS      /* see header file */
 971         SAVE_UINT32(ksi, sysinfo, rw_enters);
 972         SAVE_UINT32(ksi, sysinfo, win_uo_cnt);
 973         SAVE_UINT32(ksi, sysinfo, win_uu_cnt);
 974         SAVE_UINT32(ksi, sysinfo, win_so_cnt);
 975         SAVE_UINT32(ksi, sysinfo, win_su_cnt);
 976         SAVE_UINT32(ksi, sysinfo, win_suo_cnt);
 977 #endif
 978 
 979         SAVE_INT32(ksi, syswait, iowait);
 980         SAVE_INT32(ksi, syswait, swap);
 981         SAVE_INT32(ksi, syswait, physio);
 982 
 983         SAVE_UINT32(ksi, vminfo, pgrec);
 984         SAVE_UINT32(ksi, vminfo, pgfrec);
 985         SAVE_UINT32(ksi, vminfo, pgin);
 986         SAVE_UINT32(ksi, vminfo, pgpgin);
 987         SAVE_UINT32(ksi, vminfo, pgout);
 988         SAVE_UINT32(ksi, vminfo, pgpgout);
 989         SAVE_UINT32(ksi, vminfo, swapin);
 990         SAVE_UINT32(ksi, vminfo, pgswapin);
 991         SAVE_UINT32(ksi, vminfo, swapout);
 992         SAVE_UINT32(ksi, vminfo, pgswapout);
 993         SAVE_UINT32(ksi, vminfo, zfod);
 994         SAVE_UINT32(ksi, vminfo, dfree);
 995         SAVE_UINT32(ksi, vminfo, scan);
 996         SAVE_UINT32(ksi, vminfo, rev);
 997         SAVE_UINT32(ksi, vminfo, hat_fault);
 998         SAVE_UINT32(ksi, vminfo, as_fault);
 999         SAVE_UINT32(ksi, vminfo, maj_fault);
1000         SAVE_UINT32(ksi, vminfo, cow_fault);
1001         SAVE_UINT32(ksi, vminfo, prot_fault);
1002         SAVE_UINT32(ksi, vminfo, softlock);
1003         SAVE_UINT32(ksi, vminfo, kernel_asflt);
1004         SAVE_UINT32(ksi, vminfo, pgrrun);
1005         SAVE_UINT32(ksi, vminfo, execpgin);
1006         SAVE_UINT32(ksi, vminfo, execpgout);
1007         SAVE_UINT32(ksi, vminfo, execfree);
1008         SAVE_UINT32(ksi, vminfo, anonpgin);
1009         SAVE_UINT32(ksi, vminfo, anonpgout);
1010         SAVE_UINT32(ksi, vminfo, anonfree);
1011         SAVE_UINT32(ksi, vminfo, fspgin);
1012         SAVE_UINT32(ksi, vminfo, fspgout);
1013         SAVE_UINT32(ksi, vminfo, fsfree);
1014 }
1015 
1016 static void
1017 save_var(kstat_t *kp, ks_instance_t *ksi)
1018 {
1019         struct var      *var = (struct var *)(kp->ks_data);
1020 
1021         assert(kp->ks_data_size == sizeof (struct var));
1022 
1023         SAVE_INT32(ksi, var, v_buf);
1024         SAVE_INT32(ksi, var, v_call);
1025         SAVE_INT32(ksi, var, v_proc);
1026         SAVE_INT32(ksi, var, v_maxupttl);
1027         SAVE_INT32(ksi, var, v_nglobpris);
1028         SAVE_INT32(ksi, var, v_maxsyspri);
1029         SAVE_INT32(ksi, var, v_clist);
1030         SAVE_INT32(ksi, var, v_maxup);
1031         SAVE_INT32(ksi, var, v_hbuf);
1032         SAVE_INT32(ksi, var, v_hmask);
1033         SAVE_INT32(ksi, var, v_pbuf);
1034         SAVE_INT32(ksi, var, v_sptmap);
1035         SAVE_INT32(ksi, var, v_maxpmem);
1036         SAVE_INT32(ksi, var, v_autoup);
1037         SAVE_INT32(ksi, var, v_bufhwm);
1038 }
1039 
1040 static void
1041 save_ncstats(kstat_t *kp, ks_instance_t *ksi)
1042 {
1043         struct ncstats  *ncstats = (struct ncstats *)(kp->ks_data);
1044 
1045         assert(kp->ks_data_size == sizeof (struct ncstats));
1046 
1047         SAVE_INT32(ksi, ncstats, hits);
1048         SAVE_INT32(ksi, ncstats, misses);
1049         SAVE_INT32(ksi, ncstats, enters);
1050         SAVE_INT32(ksi, ncstats, dbl_enters);
1051         SAVE_INT32(ksi, ncstats, long_enter);
1052         SAVE_INT32(ksi, ncstats, long_look);
1053         SAVE_INT32(ksi, ncstats, move_to_front);
1054         SAVE_INT32(ksi, ncstats, purges);
1055 }
1056 
1057 static void
1058 save_sysinfo(kstat_t *kp, ks_instance_t *ksi)
1059 {
1060         sysinfo_t       *sysinfo = (sysinfo_t *)(kp->ks_data);
1061 
1062         assert(kp->ks_data_size == sizeof (sysinfo_t));
1063 
1064         SAVE_UINT32(ksi, sysinfo, updates);
1065         SAVE_UINT32(ksi, sysinfo, runque);
1066         SAVE_UINT32(ksi, sysinfo, runocc);
1067         SAVE_UINT32(ksi, sysinfo, swpque);
1068         SAVE_UINT32(ksi, sysinfo, swpocc);
1069         SAVE_UINT32(ksi, sysinfo, waiting);
1070 }
1071 
1072 static void
1073 save_vminfo(kstat_t *kp, ks_instance_t *ksi)
1074 {
1075         vminfo_t        *vminfo = (vminfo_t *)(kp->ks_data);
1076 
1077         assert(kp->ks_data_size == sizeof (vminfo_t));
1078 
1079         SAVE_UINT64(ksi, vminfo, freemem);
1080         SAVE_UINT64(ksi, vminfo, swap_resv);
1081         SAVE_UINT64(ksi, vminfo, swap_alloc);
1082         SAVE_UINT64(ksi, vminfo, swap_avail);
1083         SAVE_UINT64(ksi, vminfo, swap_free);
1084         SAVE_UINT64(ksi, vminfo, updates);
1085 }
1086 
1087 static void
1088 save_nfs(kstat_t *kp, ks_instance_t *ksi)
1089 {
1090         struct mntinfo_kstat *mntinfo = (struct mntinfo_kstat *)(kp->ks_data);
1091 
1092         assert(kp->ks_data_size == sizeof (struct mntinfo_kstat));
1093 
1094         SAVE_STRING(ksi, mntinfo, mik_proto);
1095         SAVE_UINT32(ksi, mntinfo, mik_vers);
1096         SAVE_UINT32(ksi, mntinfo, mik_flags);
1097         SAVE_UINT32(ksi, mntinfo, mik_secmod);
1098         SAVE_UINT32(ksi, mntinfo, mik_curread);
1099         SAVE_UINT32(ksi, mntinfo, mik_curwrite);
1100         SAVE_INT32(ksi, mntinfo, mik_timeo);
1101         SAVE_INT32(ksi, mntinfo, mik_retrans);
1102         SAVE_UINT32(ksi, mntinfo, mik_acregmin);
1103         SAVE_UINT32(ksi, mntinfo, mik_acregmax);
1104         SAVE_UINT32(ksi, mntinfo, mik_acdirmin);
1105         SAVE_UINT32(ksi, mntinfo, mik_acdirmax);
1106         SAVE_UINT32_X(ksi, "lookup_srtt", mntinfo->mik_timers[0].srtt);
1107         SAVE_UINT32_X(ksi, "lookup_deviate", mntinfo->mik_timers[0].deviate);
1108         SAVE_UINT32_X(ksi, "lookup_rtxcur", mntinfo->mik_timers[0].rtxcur);
1109         SAVE_UINT32_X(ksi, "read_srtt", mntinfo->mik_timers[1].srtt);
1110         SAVE_UINT32_X(ksi, "read_deviate", mntinfo->mik_timers[1].deviate);
1111         SAVE_UINT32_X(ksi, "read_rtxcur", mntinfo->mik_timers[1].rtxcur);
1112         SAVE_UINT32_X(ksi, "write_srtt", mntinfo->mik_timers[2].srtt);
1113         SAVE_UINT32_X(ksi, "write_deviate", mntinfo->mik_timers[2].deviate);
1114         SAVE_UINT32_X(ksi, "write_rtxcur", mntinfo->mik_timers[2].rtxcur);
1115         SAVE_UINT32(ksi, mntinfo, mik_noresponse);
1116         SAVE_UINT32(ksi, mntinfo, mik_failover);
1117         SAVE_UINT32(ksi, mntinfo, mik_remap);
1118         SAVE_STRING(ksi, mntinfo, mik_curserver);
1119 }
1120 
1121 #ifdef __sparc
1122 static void
1123 save_sfmmu_global_stat(kstat_t *kp, ks_instance_t *ksi)
1124 {
1125         struct sfmmu_global_stat *sfmmug =
1126             (struct sfmmu_global_stat *)(kp->ks_data);
1127 
1128         assert(kp->ks_data_size == sizeof (struct sfmmu_global_stat));
1129 
1130         SAVE_INT32(ksi, sfmmug, sf_tsb_exceptions);
1131         SAVE_INT32(ksi, sfmmug, sf_tsb_raise_exception);
1132         SAVE_INT32(ksi, sfmmug, sf_pagefaults);
1133         SAVE_INT32(ksi, sfmmug, sf_uhash_searches);
1134         SAVE_INT32(ksi, sfmmug, sf_uhash_links);
1135         SAVE_INT32(ksi, sfmmug, sf_khash_searches);
1136         SAVE_INT32(ksi, sfmmug, sf_khash_links);
1137         SAVE_INT32(ksi, sfmmug, sf_swapout);
1138         SAVE_INT32(ksi, sfmmug, sf_tsb_alloc);
1139         SAVE_INT32(ksi, sfmmug, sf_tsb_allocfail);
1140         SAVE_INT32(ksi, sfmmug, sf_tsb_sectsb_create);
1141         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_alloc);
1142         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_alloc);
1143         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_allocfail);
1144         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_allocfail);
1145         SAVE_INT32(ksi, sfmmug, sf_tteload8k);
1146         SAVE_INT32(ksi, sfmmug, sf_tteload64k);
1147         SAVE_INT32(ksi, sfmmug, sf_tteload512k);
1148         SAVE_INT32(ksi, sfmmug, sf_tteload4m);
1149         SAVE_INT32(ksi, sfmmug, sf_tteload32m);
1150         SAVE_INT32(ksi, sfmmug, sf_tteload256m);
1151         SAVE_INT32(ksi, sfmmug, sf_tsb_load8k);
1152         SAVE_INT32(ksi, sfmmug, sf_tsb_load4m);
1153         SAVE_INT32(ksi, sfmmug, sf_hblk_hit);
1154         SAVE_INT32(ksi, sfmmug, sf_hblk8_ncreate);
1155         SAVE_INT32(ksi, sfmmug, sf_hblk8_nalloc);
1156         SAVE_INT32(ksi, sfmmug, sf_hblk1_ncreate);
1157         SAVE_INT32(ksi, sfmmug, sf_hblk1_nalloc);
1158         SAVE_INT32(ksi, sfmmug, sf_hblk_slab_cnt);
1159         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_cnt);
1160         SAVE_INT32(ksi, sfmmug, sf_hblk_recurse_cnt);
1161         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_hit);
1162         SAVE_INT32(ksi, sfmmug, sf_get_free_success);
1163         SAVE_INT32(ksi, sfmmug, sf_get_free_throttle);
1164         SAVE_INT32(ksi, sfmmug, sf_get_free_fail);
1165         SAVE_INT32(ksi, sfmmug, sf_put_free_success);
1166         SAVE_INT32(ksi, sfmmug, sf_put_free_fail);
1167         SAVE_INT32(ksi, sfmmug, sf_pgcolor_conflict);
1168         SAVE_INT32(ksi, sfmmug, sf_uncache_conflict);
1169         SAVE_INT32(ksi, sfmmug, sf_unload_conflict);
1170         SAVE_INT32(ksi, sfmmug, sf_ism_uncache);
1171         SAVE_INT32(ksi, sfmmug, sf_ism_recache);
1172         SAVE_INT32(ksi, sfmmug, sf_recache);
1173         SAVE_INT32(ksi, sfmmug, sf_steal_count);
1174         SAVE_INT32(ksi, sfmmug, sf_pagesync);
1175         SAVE_INT32(ksi, sfmmug, sf_clrwrt);
1176         SAVE_INT32(ksi, sfmmug, sf_pagesync_invalid);
1177         SAVE_INT32(ksi, sfmmug, sf_kernel_xcalls);
1178         SAVE_INT32(ksi, sfmmug, sf_user_xcalls);
1179         SAVE_INT32(ksi, sfmmug, sf_tsb_grow);
1180         SAVE_INT32(ksi, sfmmug, sf_tsb_shrink);
1181         SAVE_INT32(ksi, sfmmug, sf_tsb_resize_failures);
1182         SAVE_INT32(ksi, sfmmug, sf_tsb_reloc);
1183         SAVE_INT32(ksi, sfmmug, sf_user_vtop);
1184         SAVE_INT32(ksi, sfmmug, sf_ctx_inv);
1185         SAVE_INT32(ksi, sfmmug, sf_tlb_reprog_pgsz);
1186         SAVE_INT32(ksi, sfmmug, sf_region_remap_demap);
1187         SAVE_INT32(ksi, sfmmug, sf_create_scd);
1188         SAVE_INT32(ksi, sfmmug, sf_join_scd);
1189         SAVE_INT32(ksi, sfmmug, sf_leave_scd);
1190         SAVE_INT32(ksi, sfmmug, sf_destroy_scd);
1191 }
1192 #endif
1193 
1194 #ifdef __sparc
1195 static void
1196 save_sfmmu_tsbsize_stat(kstat_t *kp, ks_instance_t *ksi)
1197 {
1198         struct sfmmu_tsbsize_stat *sfmmut;
1199 
1200         assert(kp->ks_data_size == sizeof (struct sfmmu_tsbsize_stat));
1201         sfmmut = (struct sfmmu_tsbsize_stat *)(kp->ks_data);
1202 
1203         SAVE_INT32(ksi, sfmmut, sf_tsbsz_8k);
1204         SAVE_INT32(ksi, sfmmut, sf_tsbsz_16k);
1205         SAVE_INT32(ksi, sfmmut, sf_tsbsz_32k);
1206         SAVE_INT32(ksi, sfmmut, sf_tsbsz_64k);
1207         SAVE_INT32(ksi, sfmmut, sf_tsbsz_128k);
1208         SAVE_INT32(ksi, sfmmut, sf_tsbsz_256k);
1209         SAVE_INT32(ksi, sfmmut, sf_tsbsz_512k);
1210         SAVE_INT32(ksi, sfmmut, sf_tsbsz_1m);
1211         SAVE_INT32(ksi, sfmmut, sf_tsbsz_2m);
1212         SAVE_INT32(ksi, sfmmut, sf_tsbsz_4m);
1213 }
1214 #endif
1215 
1216 #ifdef __sparc
1217 static void
1218 save_simmstat(kstat_t *kp, ks_instance_t *ksi)
1219 {
1220         uchar_t *simmstat;
1221         char    *simm_buf;
1222         char    *list = NULL;
1223         int     i;
1224 
1225         assert(kp->ks_data_size == sizeof (uchar_t) * SIMM_COUNT);
1226 
1227         for (i = 0, simmstat = (uchar_t *)(kp->ks_data); i < SIMM_COUNT - 1;
1228             i++, simmstat++) {
1229                 if (list == NULL) {
1230                         (void) asprintf(&simm_buf, "%d,", *simmstat);
1231                 } else {
1232                         (void) asprintf(&simm_buf, "%s%d,", list, *simmstat);
1233                         free(list);
1234                 }
1235                 list = simm_buf;
1236         }
1237 
1238         (void) asprintf(&simm_buf, "%s%d", list, *simmstat);
1239         SAVE_STRING_X(ksi, "status", simm_buf);
1240         free(list);
1241         free(simm_buf);
1242 }
1243 #endif
1244 
1245 #ifdef __sparc
1246 /*
1247  * Helper function for save_temperature().
1248  */
1249 static char *
1250 short_array_to_string(short *shortp, int len)
1251 {
1252         char    *list = NULL;
1253         char    *list_buf;
1254 
1255         for (; len > 1; len--, shortp++) {
1256                 if (list == NULL) {
1257                         (void) asprintf(&list_buf, "%hd,", *shortp);
1258                 } else {
1259                         (void) asprintf(&list_buf, "%s%hd,", list, *shortp);
1260                         free(list);
1261                 }
1262                 list = list_buf;
1263         }
1264 
1265         (void) asprintf(&list_buf, "%s%hd", list, *shortp);
1266         free(list);
1267         return (list_buf);
1268 }
1269 
1270 static void
1271 save_temperature(kstat_t *kp, ks_instance_t *ksi)
1272 {
1273         struct temp_stats *temps = (struct temp_stats *)(kp->ks_data);
1274         char    *buf;
1275 
1276         assert(kp->ks_data_size == sizeof (struct temp_stats));
1277 
1278         SAVE_UINT32(ksi, temps, index);
1279 
1280         buf = short_array_to_string(temps->l1, L1_SZ);
1281         SAVE_STRING_X(ksi, "l1", buf);
1282         free(buf);
1283 
1284         buf = short_array_to_string(temps->l2, L2_SZ);
1285         SAVE_STRING_X(ksi, "l2", buf);
1286         free(buf);
1287 
1288         buf = short_array_to_string(temps->l3, L3_SZ);
1289         SAVE_STRING_X(ksi, "l3", buf);
1290         free(buf);
1291 
1292         buf = short_array_to_string(temps->l4, L4_SZ);
1293         SAVE_STRING_X(ksi, "l4", buf);
1294         free(buf);
1295 
1296         buf = short_array_to_string(temps->l5, L5_SZ);
1297         SAVE_STRING_X(ksi, "l5", buf);
1298         free(buf);
1299 
1300         SAVE_INT32(ksi, temps, max);
1301         SAVE_INT32(ksi, temps, min);
1302         SAVE_INT32(ksi, temps, state);
1303         SAVE_INT32(ksi, temps, temp_cnt);
1304         SAVE_INT32(ksi, temps, shutdown_cnt);
1305         SAVE_INT32(ksi, temps, version);
1306         SAVE_INT32(ksi, temps, trend);
1307         SAVE_INT32(ksi, temps, override);
1308 }
1309 #endif
1310 
1311 #ifdef __sparc
1312 static void
1313 save_temp_over(kstat_t *kp, ks_instance_t *ksi)
1314 {
1315         short   *sh = (short *)(kp->ks_data);
1316         char    *value;
1317 
1318         assert(kp->ks_data_size == sizeof (short));
1319 
1320         (void) asprintf(&value, "%hu", *sh);
1321         SAVE_STRING_X(ksi, "override", value);
1322         free(value);
1323 }
1324 #endif
1325 
1326 #ifdef __sparc
1327 static void
1328 save_ps_shadow(kstat_t *kp, ks_instance_t *ksi)
1329 {
1330         uchar_t *uchar = (uchar_t *)(kp->ks_data);
1331 
1332         assert(kp->ks_data_size == SYS_PS_COUNT);
1333 
1334         SAVE_CHAR_X(ksi, "core_0", *uchar++);
1335         SAVE_CHAR_X(ksi, "core_1", *uchar++);
1336         SAVE_CHAR_X(ksi, "core_2", *uchar++);
1337         SAVE_CHAR_X(ksi, "core_3", *uchar++);
1338         SAVE_CHAR_X(ksi, "core_4", *uchar++);
1339         SAVE_CHAR_X(ksi, "core_5", *uchar++);
1340         SAVE_CHAR_X(ksi, "core_6", *uchar++);
1341         SAVE_CHAR_X(ksi, "core_7", *uchar++);
1342         SAVE_CHAR_X(ksi, "pps_0", *uchar++);
1343         SAVE_CHAR_X(ksi, "clk_33", *uchar++);
1344         SAVE_CHAR_X(ksi, "clk_50", *uchar++);
1345         SAVE_CHAR_X(ksi, "v5_p", *uchar++);
1346         SAVE_CHAR_X(ksi, "v12_p", *uchar++);
1347         SAVE_CHAR_X(ksi, "v5_aux", *uchar++);
1348         SAVE_CHAR_X(ksi, "v5_p_pch", *uchar++);
1349         SAVE_CHAR_X(ksi, "v12_p_pch", *uchar++);
1350         SAVE_CHAR_X(ksi, "v3_pch", *uchar++);
1351         SAVE_CHAR_X(ksi, "v5_pch", *uchar++);
1352         SAVE_CHAR_X(ksi, "p_fan", *uchar++);
1353 }
1354 #endif
1355 
1356 #ifdef __sparc
1357 static void
1358 save_fault_list(kstat_t *kp, ks_instance_t *ksi)
1359 {
1360         struct ft_list *fault;
1361         char    name[KSTAT_STRLEN + 7];
1362         int     i;
1363 
1364         for (i = 1, fault = (struct ft_list *)(kp->ks_data);
1365             i <= 999999 && i <= kp->ks_data_size / sizeof (struct ft_list);
1366             i++, fault++) {
1367                 (void) snprintf(name, sizeof (name), "unit_%d", i);
1368                 SAVE_INT32_X(ksi, name, fault->unit);
1369                 (void) snprintf(name, sizeof (name), "type_%d", i);
1370                 SAVE_INT32_X(ksi, name, fault->type);
1371                 (void) snprintf(name, sizeof (name), "fclass_%d", i);
1372                 SAVE_INT32_X(ksi, name, fault->fclass);
1373                 (void) snprintf(name, sizeof (name), "create_time_%d", i);
1374                 SAVE_HRTIME_X(ksi, name, fault->create_time);
1375                 (void) snprintf(name, sizeof (name), "msg_%d", i);
1376                 SAVE_STRING_X(ksi, name, fault->msg);
1377         }
1378 }
1379 #endif
1380 
1381 static void
1382 save_named(kstat_t *kp, ks_instance_t *ksi)
1383 {
1384         kstat_named_t *knp;
1385         int     n;
1386 
1387         for (n = kp->ks_ndata, knp = KSTAT_NAMED_PTR(kp); n > 0; n--, knp++) {
1388                 switch (knp->data_type) {
1389                 case KSTAT_DATA_CHAR:
1390                         nvpair_insert(ksi, knp->name,
1391                             (ks_value_t *)&knp->value, KSTAT_DATA_CHAR);
1392                         break;
1393                 case KSTAT_DATA_INT32:
1394                         nvpair_insert(ksi, knp->name,
1395                             (ks_value_t *)&knp->value, KSTAT_DATA_INT32);
1396                         break;
1397                 case KSTAT_DATA_UINT32:
1398                         nvpair_insert(ksi, knp->name,
1399                             (ks_value_t *)&knp->value, KSTAT_DATA_UINT32);
1400                         break;
1401                 case KSTAT_DATA_INT64:
1402                         nvpair_insert(ksi, knp->name,
1403                             (ks_value_t *)&knp->value, KSTAT_DATA_INT64);
1404                         break;
1405                 case KSTAT_DATA_UINT64:
1406                         nvpair_insert(ksi, knp->name,
1407                             (ks_value_t *)&knp->value, KSTAT_DATA_UINT64);
1408                         break;
1409                 case KSTAT_DATA_STRING:
1410                         SAVE_STRING_X(ksi, knp->name, KSTAT_NAMED_STR_PTR(knp));
1411                         break;
1412                 default:
1413                         assert(B_FALSE); /* Invalid data type */
1414                         break;
1415                 }
1416         }
1417 }
1418 
1419 static void
1420 save_intr(kstat_t *kp, ks_instance_t *ksi)
1421 {
1422         kstat_intr_t *intr = KSTAT_INTR_PTR(kp);
1423         char    *intr_names[] = {"hard", "soft", "watchdog", "spurious",
1424             "multiple_service"};
1425         int     n;
1426 
1427         for (n = 0; n < KSTAT_NUM_INTRS; n++)
1428                 SAVE_UINT32_X(ksi, intr_names[n], intr->intrs[n]);
1429 }
1430 
1431 static void
1432 save_io(kstat_t *kp, ks_instance_t *ksi)
1433 {
1434         kstat_io_t      *ksio = KSTAT_IO_PTR(kp);
1435 
1436         SAVE_UINT64(ksi, ksio, nread);
1437         SAVE_UINT64(ksi, ksio, nwritten);
1438         SAVE_UINT32(ksi, ksio, reads);
1439         SAVE_UINT32(ksi, ksio, writes);
1440         SAVE_HRTIME(ksi, ksio, wtime);
1441         SAVE_HRTIME(ksi, ksio, wlentime);
1442         SAVE_HRTIME(ksi, ksio, wlastupdate);
1443         SAVE_HRTIME(ksi, ksio, rtime);
1444         SAVE_HRTIME(ksi, ksio, rlentime);
1445         SAVE_HRTIME(ksi, ksio, rlastupdate);
1446         SAVE_UINT32(ksi, ksio, wcnt);
1447         SAVE_UINT32(ksi, ksio, rcnt);
1448 }
1449 
1450 static void
1451 save_timer(kstat_t *kp, ks_instance_t *ksi)
1452 {
1453         kstat_timer_t   *ktimer = KSTAT_TIMER_PTR(kp);
1454 
1455         SAVE_STRING(ksi, ktimer, name);
1456         SAVE_UINT64(ksi, ktimer, num_events);
1457         SAVE_HRTIME(ksi, ktimer, elapsed_time);
1458         SAVE_HRTIME(ksi, ktimer, min_time);
1459         SAVE_HRTIME(ksi, ktimer, max_time);
1460         SAVE_HRTIME(ksi, ktimer, start_time);
1461         SAVE_HRTIME(ksi, ktimer, stop_time);
1462 }