1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013 David Hoeppner. All rights reserved.
  25  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
  26  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  27  * Copyright 2016 Garrett D'Amore
  28  */
  29 
  30 /*
  31  * Display kernel statistics
  32  *
  33  * This is a reimplementation of the perl kstat command originally found
  34  * under usr/src/cmd/kstat/kstat.pl
  35  *
  36  * Incompatibilities:
  37  *      - perl regular expressions replaced with extended REs bracketed by '/'
  38  *
  39  * Flags added:
  40  *      -C      similar to the -p option but value is separated by a colon
  41  *      -h      display help
  42  *      -j      json format
  43  *      -H b    display hrtime as seconds since boot
  44  *      -H d    display hrtime as date
  45  *      -H u    display hrtime as UNIX seconds (since epoch)
  46  *      -H I    display hrtime as ISO-8601 local time
  47  *      -H Z    display hrtime as ISO-8601 UTC
  48  *      -H n    display hrtime as nanoseconds (since epoch)
  49  */
  50 
  51 #include <assert.h>
  52 #include <ctype.h>
  53 #include <errno.h>
  54 #include <kstat.h>
  55 #include <langinfo.h>
  56 #include <libgen.h>
  57 #include <limits.h>
  58 #include <locale.h>
  59 #include <signal.h>
  60 #include <stddef.h>
  61 #include <stdio.h>
  62 #include <stdlib.h>
  63 #include <string.h>
  64 #include <strings.h>
  65 #include <time.h>
  66 #include <unistd.h>
  67 #include <sys/list.h>
  68 #include <sys/time.h>
  69 #include <sys/types.h>
  70 
  71 #include "kstat.h"
  72 #include "statcommon.h"
  73 
  74 char    *cmdname = "kstat";     /* Name of this command */
  75 int     caught_cont = 0;        /* Have caught a SIGCONT */
  76 
  77 static uint_t   g_timestamp_fmt = NODATE;
  78 static uint_t   g_hrtime_fmt = KS_HRFMT_DEFAULT;
  79 
  80 /* Helper flag - header was printed already? */
  81 static boolean_t g_headerflg;
  82 
  83 /* Saved command line options */
  84 static boolean_t g_cflg = B_FALSE;
  85 static boolean_t g_jflg = B_FALSE;
  86 static boolean_t g_lflg = B_FALSE;
  87 static boolean_t g_pflg = B_FALSE;
  88 static boolean_t g_qflg = B_FALSE;
  89 static ks_pattern_t     g_ks_class = {"*", 0};
  90 
  91 /* Return zero if a selector did match */
  92 static int      g_matched = 1;
  93 
  94 /* Sorted list of kstat instances */
  95 static list_t   instances_list;
  96 static list_t   selector_list;
  97 
  98 static hrtime_t hrtime_origin;
  99 
 100 int
 101 main(int argc, char **argv)
 102 {
 103         ks_selector_t   *nselector;
 104         ks_selector_t   *uselector;
 105         kstat_ctl_t     *kc;
 106         hrtime_t        start_n;
 107         hrtime_t        period_n;
 108         boolean_t       errflg = B_FALSE;
 109         boolean_t       nselflg = B_FALSE;
 110         boolean_t       uselflg = B_FALSE;
 111         char            *q;
 112         int             count = 1;
 113         int             infinite_cycles = 0;
 114         int             interval = 0;
 115         int             n = 0;
 116         int             c, m, tmp;
 117 
 118         (void) setlocale(LC_ALL, "");
 119 #if !defined(TEXT_DOMAIN)               /* Should be defined by cc -D */
 120 #define TEXT_DOMAIN "SYS_TEST"          /* Use this only if it wasn't */
 121 #endif
 122         (void) textdomain(TEXT_DOMAIN);
 123 
 124         /*
 125          * Create the selector list and a dummy default selector to match
 126          * everything. While we process the cmdline options we will add
 127          * selectors to this list.
 128          */
 129         list_create(&selector_list, sizeof (ks_selector_t),
 130             offsetof(ks_selector_t, ks_next));
 131 
 132         nselector = new_selector();
 133 
 134         /*
 135          * Parse named command line arguments.
 136          */
 137         while ((c = getopt(argc, argv, "h?CqjlpT:m:i:n:s:c:H:")) != EOF)
 138                 switch (c) {
 139                 case 'h':
 140                 case '?':
 141                         usage();
 142                         exit(0);
 143                         break;
 144                 case 'C':
 145                         g_pflg = g_cflg = B_TRUE;
 146                         break;
 147                 case 'q':
 148                         g_qflg = B_TRUE;
 149                         break;
 150                 case 'j':
 151                         g_jflg = B_TRUE;
 152                         break;
 153                 case 'l':
 154                         g_pflg = g_lflg = B_TRUE;
 155                         break;
 156                 case 'p':
 157                         g_pflg = B_TRUE;
 158                         break;
 159                 case 'T':
 160                         if (strlen(optarg) != 1) {
 161                                 errflg = B_TRUE;
 162                         }
 163                         switch (*optarg) {
 164                         case 'd':
 165                                 g_timestamp_fmt = DDATE;
 166                                 break;
 167                         case 'u':
 168                                 g_timestamp_fmt = UDATE;
 169                                 break;
 170                         default:
 171                                 errflg = B_TRUE;
 172                         }
 173                         break;
 174                 case 'm':
 175                         nselflg = B_TRUE;
 176                         nselector->ks_module.pstr =
 177                             (char *)ks_safe_strdup(optarg);
 178                         break;
 179                 case 'i':
 180                         nselflg = B_TRUE;
 181                         nselector->ks_instance.pstr =
 182                             (char *)ks_safe_strdup(optarg);
 183                         break;
 184                 case 'n':
 185                         nselflg = B_TRUE;
 186                         nselector->ks_name.pstr =
 187                             (char *)ks_safe_strdup(optarg);
 188                         break;
 189                 case 's':
 190                         nselflg = B_TRUE;
 191                         nselector->ks_statistic.pstr =
 192                             (char *)ks_safe_strdup(optarg);
 193                         break;
 194                 case 'c':
 195                         g_ks_class.pstr =
 196                             (char *)ks_safe_strdup(optarg);
 197                         break;
 198                 case 'H':
 199                         switch (*optarg) {
 200                         case 'o':
 201                         case 'b':
 202                         case 'n':
 203                         case 'I':
 204                         case 'Z':
 205                         case 'd':
 206                         case 'u':
 207                                 if (strlen(optarg) != 1) {
 208                                         errflg = B_TRUE;
 209                                 }
 210                                 g_hrtime_fmt = *optarg;
 211                                 break;
 212                         default:
 213                                 errflg = B_TRUE;
 214                                 break;
 215                         }
 216                         break;
 217                 default:
 218                         errflg = B_TRUE;
 219                         break;
 220                 }
 221 
 222         if (g_qflg && (g_jflg || g_pflg)) {
 223                 (void) fprintf(stderr, gettext(
 224                     "-q and -lpj are mutually exclusive\n"));
 225                 errflg = B_TRUE;
 226         }
 227 
 228         switch (g_hrtime_fmt) {
 229         case KS_HRFMT_DEFAULT:
 230         case KS_HRFMT_BOOT:
 231                 break;
 232         default: {
 233                 struct timespec ts;
 234                 if (clock_gettime(CLOCK_REALTIME, &ts) != 0) {
 235                         perror("clock_gettime(CLOCK_REALTIME)");
 236                         exit(3);
 237                 }
 238                 hrtime_origin = ts.tv_sec;
 239                 hrtime_origin *= 1000000000;
 240                 hrtime_origin += ts.tv_nsec;
 241                 hrtime_origin -= gethrtime();
 242         }
 243         }
 244 
 245         if (errflg) {
 246                 usage();
 247                 exit(2);
 248         }
 249 
 250         argc -= optind;
 251         argv += optind;
 252 
 253         /*
 254          * Consume the rest of the command line. Parsing the
 255          * unnamed command line arguments.
 256          */
 257         while (argc--) {
 258                 errno = 0;
 259                 tmp = strtoul(*argv, &q, 10);
 260                 if (tmp == ULONG_MAX && errno == ERANGE) {
 261                         if (n == 0) {
 262                                 (void) fprintf(stderr, gettext(
 263                                     "Interval is too large\n"));
 264                         } else if (n == 1) {
 265                                 (void) fprintf(stderr, gettext(
 266                                     "Count is too large\n"));
 267                         }
 268                         usage();
 269                         exit(2);
 270                 }
 271 
 272                 if (errno != 0 || *q != '\0') {
 273                         m = 0;
 274                         uselector = new_selector();
 275                         while ((q = (char *)strsep(argv, ":")) != NULL) {
 276                                 m++;
 277                                 if (m > 4) {
 278                                         free(uselector);
 279                                         usage();
 280                                         exit(2);
 281                                 }
 282 
 283                                 if (*q != '\0') {
 284                                         switch (m) {
 285                                         case 1:
 286                                                 uselector->ks_module.pstr =
 287                                                     (char *)ks_safe_strdup(q);
 288                                                 break;
 289                                         case 2:
 290                                                 uselector->ks_instance.pstr =
 291                                                     (char *)ks_safe_strdup(q);
 292                                                 break;
 293                                         case 3:
 294                                                 uselector->ks_name.pstr =
 295                                                     (char *)ks_safe_strdup(q);
 296                                                 break;
 297                                         case 4:
 298                                                 uselector->ks_statistic.pstr =
 299                                                     (char *)ks_safe_strdup(q);
 300                                                 break;
 301                                         default:
 302                                                 assert(B_FALSE);
 303                                         }
 304                                 }
 305                         }
 306 
 307                         uselflg = B_TRUE;
 308                         list_insert_tail(&selector_list, uselector);
 309                 } else {
 310                         if (tmp < 1) {
 311                                 if (n == 0) {
 312                                         (void) fprintf(stderr, gettext(
 313                                             "Interval must be an "
 314                                             "integer >= 1"));
 315                                 } else if (n == 1) {
 316                                         (void) fprintf(stderr, gettext(
 317                                             "Count must be an integer >= 1"));
 318                                 }
 319                                 usage();
 320                                 exit(2);
 321                         } else {
 322                                 if (n == 0) {
 323                                         interval = tmp;
 324                                         count = -1;
 325                                 } else if (n == 1) {
 326                                         count = tmp;
 327                                 } else {
 328                                         usage();
 329                                         exit(2);
 330                                 }
 331                         }
 332                         n++;
 333                 }
 334                 argv++;
 335         }
 336 
 337         /*
 338          * Check if we founded a named selector on the cmdline.
 339          */
 340         if (uselflg) {
 341                 if (nselflg) {
 342                         (void) fprintf(stderr, gettext(
 343                             "[module[:instance[:name[:statistic]]]] and "
 344                             "-m -i -n -s are mutually exclusive"));
 345                         usage();
 346                         exit(2);
 347                 } else {
 348                         free(nselector);
 349                 }
 350         } else {
 351                 list_insert_tail(&selector_list, nselector);
 352         }
 353 
 354         assert(!list_is_empty(&selector_list));
 355 
 356         list_create(&instances_list, sizeof (ks_instance_t),
 357             offsetof(ks_instance_t, ks_next));
 358 
 359         while ((kc = kstat_open()) == NULL) {
 360                 if (errno == EAGAIN) {
 361                         (void) poll(NULL, 0, 200);
 362                 } else {
 363                         perror("kstat_open");
 364                         exit(3);
 365                 }
 366         }
 367 
 368         if (count > 1) {
 369                 if (signal(SIGCONT, cont_handler) == SIG_ERR) {
 370                         (void) fprintf(stderr, gettext(
 371                             "signal failed"));
 372                         exit(3);
 373                 }
 374         }
 375 
 376         period_n = (hrtime_t)interval * NANOSEC;
 377         start_n = gethrtime();
 378 
 379         while (count == -1 || count-- > 0) {
 380                 ks_instances_read(kc);
 381                 ks_instances_print();
 382 
 383                 if (interval && count) {
 384                         ks_sleep_until(&start_n, period_n, infinite_cycles,
 385                             &caught_cont);
 386                         (void) kstat_chain_update(kc);
 387                         (void) putchar('\n');
 388                 }
 389         }
 390 
 391         (void) kstat_close(kc);
 392 
 393         return (g_matched);
 394 }
 395 
 396 /*
 397  * Print usage.
 398  */
 399 static void
 400 usage(void)
 401 {
 402         (void) fprintf(stderr, gettext(
 403             "Usage:\n"
 404             "kstat [ -Cjlpq ] [ -T d|u ] [ -H d|n|u|I|Z ] [ -c class ]\n"
 405             "      [ -m module ] [ -i instance ] [ -n name ] [ -s statistic ]\n"
 406             "      [ interval [ count ] ]\n"
 407             "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ] [ -H d|n|u|I|Z ]\n"
 408             "      [ module[:instance[:name[:statistic]]] ... ]\n"
 409             "      [ interval [ count ] ]\n"));
 410 }
 411 
 412 /*
 413  * Sort compare function.
 414  */
 415 static int
 416 compare_instances(ks_instance_t *l_arg, ks_instance_t *r_arg)
 417 {
 418         int     rval;
 419 
 420         rval = strcasecmp(l_arg->ks_module, r_arg->ks_module);
 421         if (rval == 0) {
 422                 if (l_arg->ks_instance == r_arg->ks_instance) {
 423                         return (strcasecmp(l_arg->ks_name, r_arg->ks_name));
 424                 } else if (l_arg->ks_instance < r_arg->ks_instance) {
 425                         return (-1);
 426                 } else {
 427                         return (1);
 428                 }
 429         } else {
 430                 return (rval);
 431         }
 432 }
 433 
 434 static char *
 435 ks_safe_strdup(char *str)
 436 {
 437         char    *ret;
 438 
 439         if (str == NULL) {
 440                 return (NULL);
 441         }
 442 
 443         while ((ret = strdup(str)) == NULL) {
 444                 if (errno == EAGAIN) {
 445                         (void) poll(NULL, 0, 200);
 446                 } else {
 447                         perror("strdup");
 448                         exit(3);
 449                 }
 450         }
 451 
 452         return (ret);
 453 }
 454 
 455 static void
 456 ks_sleep_until(hrtime_t *wakeup, hrtime_t interval, int forever,
 457     int *caught_cont)
 458 {
 459         hrtime_t        now, pause, pause_left;
 460         struct timespec pause_tv;
 461         int             status;
 462 
 463         now = gethrtime();
 464         pause = *wakeup + interval - now;
 465 
 466         if (pause <= 0 || pause < (interval / 4)) {
 467                 if (forever || *caught_cont) {
 468                         *wakeup = now + interval;
 469                         pause = interval;
 470                 } else {
 471                         pause = interval / 2;
 472                         *wakeup += interval;
 473                 }
 474         } else {
 475                 *wakeup += interval;
 476         }
 477 
 478         if (pause < 1000) {
 479                 return;
 480         }
 481 
 482         pause_left = pause;
 483         do {
 484                 pause_tv.tv_sec = pause_left / NANOSEC;
 485                 pause_tv.tv_nsec = pause_left % NANOSEC;
 486                 status = nanosleep(&pause_tv, (struct timespec *)NULL);
 487                 if (status < 0) {
 488                         if (errno == EINTR) {
 489                                 now = gethrtime();
 490                                 pause_left = *wakeup - now;
 491                                 if (pause_left < 1000) {
 492                                         return;
 493                                 }
 494                         } else {
 495                                 perror("nanosleep");
 496                                 exit(3);
 497                         }
 498                 }
 499         } while (status != 0);
 500 }
 501 
 502 /*
 503  * Inserts an instance in the per selector list.
 504  */
 505 static void
 506 nvpair_insert(ks_instance_t *ksi, char *name, ks_value_t *value,
 507     uchar_t data_type)
 508 {
 509         ks_nvpair_t     *instance;
 510         ks_nvpair_t     *tmp;
 511 
 512         instance = (ks_nvpair_t *)malloc(sizeof (ks_nvpair_t));
 513         if (instance == NULL) {
 514                 perror("malloc");
 515                 exit(3);
 516         }
 517 
 518         (void) strlcpy(instance->name, name, KSTAT_STRLEN);
 519         (void) memcpy(&instance->value, value, sizeof (ks_value_t));
 520         instance->data_type = data_type;
 521 
 522         tmp = list_head(&ksi->ks_nvlist);
 523         while (tmp != NULL && strcasecmp(instance->name, tmp->name) > 0)
 524                 tmp = list_next(&ksi->ks_nvlist, tmp);
 525 
 526         list_insert_before(&ksi->ks_nvlist, tmp, instance);
 527 }
 528 
 529 /*
 530  * Allocates a new all-matching selector.
 531  */
 532 static ks_selector_t *
 533 new_selector(void)
 534 {
 535         ks_selector_t   *selector;
 536 
 537         selector = (ks_selector_t *)malloc(sizeof (ks_selector_t));
 538         if (selector == NULL) {
 539                 perror("malloc");
 540                 exit(3);
 541         }
 542 
 543         list_link_init(&selector->ks_next);
 544 
 545         selector->ks_module.pstr = "*";
 546         selector->ks_instance.pstr = "*";
 547         selector->ks_name.pstr = "*";
 548         selector->ks_statistic.pstr = "*";
 549 
 550         return (selector);
 551 }
 552 
 553 /*
 554  * This function was taken from the perl kstat module code - please
 555  * see for further comments there.
 556  */
 557 static kstat_raw_reader_t
 558 lookup_raw_kstat_fn(char *module, char *name)
 559 {
 560         char            key[KSTAT_STRLEN * 2];
 561         register char   *f, *t;
 562         int             n = 0;
 563 
 564         for (f = module, t = key; *f != '\0'; f++, t++) {
 565                 while (*f != '\0' && isdigit(*f))
 566                         f++;
 567                 *t = *f;
 568         }
 569         *t++ = ':';
 570 
 571         for (f = name; *f != '\0'; f++, t++) {
 572                 while (*f != '\0' && isdigit(*f))
 573                         f++;
 574                 *t = *f;
 575         }
 576         *t = '\0';
 577 
 578         while (ks_raw_lookup[n].fn != NULL) {
 579                 if (strncmp(ks_raw_lookup[n].name, key, strlen(key)) == 0)
 580                         return (ks_raw_lookup[n].fn);
 581                 n++;
 582         }
 583 
 584         return (0);
 585 }
 586 
 587 /*
 588  * Match a string against a shell glob or extended regular expression.
 589  */
 590 static boolean_t
 591 ks_match(const char *str, ks_pattern_t *pattern)
 592 {
 593         int     regcode;
 594         char    *regstr;
 595         char    *errbuf;
 596         size_t  bufsz;
 597 
 598         if (pattern->pstr != NULL && gmatch(pattern->pstr, "/*/") != 0) {
 599                 /* All regex patterns are strdup'd copies */
 600                 regstr = pattern->pstr + 1;
 601                 *(strrchr(regstr, '/')) = '\0';
 602 
 603                 regcode = regcomp(&pattern->preg, regstr,
 604                     REG_EXTENDED | REG_NOSUB);
 605                 if (regcode != 0) {
 606                         bufsz = regerror(regcode, NULL, NULL, 0);
 607                         if (bufsz != 0) {
 608                                 errbuf = malloc(bufsz);
 609                                 if (errbuf == NULL) {
 610                                         perror("malloc");
 611                                         exit(3);
 612                                 }
 613                                 (void) regerror(regcode, NULL, errbuf, bufsz);
 614                                 (void) fprintf(stderr, "kstat: %s\n", errbuf);
 615                         }
 616                         usage();
 617                         exit(2);
 618                 }
 619 
 620                 pattern->pstr = NULL;
 621         }
 622 
 623         if (pattern->pstr == NULL) {
 624                 return (regexec(&pattern->preg, str, 0, NULL, 0) == 0);
 625         }
 626 
 627         return ((gmatch(str, pattern->pstr) != 0));
 628 }
 629 
 630 /*
 631  * Iterate over all kernel statistics and save matches.
 632  */
 633 static void
 634 ks_instances_read(kstat_ctl_t *kc)
 635 {
 636         kstat_raw_reader_t save_raw = NULL;
 637         kid_t           id;
 638         ks_selector_t   *selector;
 639         ks_instance_t   *ksi;
 640         ks_instance_t   *tmp;
 641         kstat_t         *kp;
 642         boolean_t       skip;
 643 
 644         for (kp = kc->kc_chain; kp != NULL; kp = kp->ks_next) {
 645                 /* Don't bother storing the kstat headers */
 646                 if (strncmp(kp->ks_name, "kstat_", 6) == 0) {
 647                         continue;
 648                 }
 649 
 650                 /* Don't bother storing raw stats we don't understand */
 651                 if (kp->ks_type == KSTAT_TYPE_RAW) {
 652                         save_raw = lookup_raw_kstat_fn(kp->ks_module,
 653                             kp->ks_name);
 654                         if (save_raw == NULL) {
 655 #ifdef REPORT_UNKNOWN
 656                                 (void) fprintf(stderr,
 657                                     "Unknown kstat type %s:%d:%s - "
 658                                     "%d of size %d\n", kp->ks_module,
 659                                     kp->ks_instance, kp->ks_name,
 660                                     kp->ks_ndata, kp->ks_data_size);
 661 #endif
 662                                 continue;
 663                         }
 664                 }
 665 
 666                 /*
 667                  * Iterate over the list of selectors and skip
 668                  * instances we dont want. We filter for statistics
 669                  * later, as we dont know them yet.
 670                  */
 671                 skip = B_TRUE;
 672                 selector = list_head(&selector_list);
 673                 while (selector != NULL) {
 674                         if (ks_match(kp->ks_module, &selector->ks_module) &&
 675                             ks_match(kp->ks_name, &selector->ks_name)) {
 676                                 skip = B_FALSE;
 677                                 break;
 678                         }
 679                         selector = list_next(&selector_list, selector);
 680                 }
 681 
 682                 if (skip) {
 683                         continue;
 684                 }
 685 
 686                 /*
 687                  * Allocate a new instance and fill in the values
 688                  * we know so far.
 689                  */
 690                 ksi = (ks_instance_t *)malloc(sizeof (ks_instance_t));
 691                 if (ksi == NULL) {
 692                         perror("malloc");
 693                         exit(3);
 694                 }
 695 
 696                 list_link_init(&ksi->ks_next);
 697 
 698                 (void) strlcpy(ksi->ks_module, kp->ks_module, KSTAT_STRLEN);
 699                 (void) strlcpy(ksi->ks_name, kp->ks_name, KSTAT_STRLEN);
 700                 (void) strlcpy(ksi->ks_class, kp->ks_class, KSTAT_STRLEN);
 701 
 702                 ksi->ks_instance = kp->ks_instance;
 703                 ksi->ks_snaptime = kp->ks_snaptime;
 704                 ksi->ks_type = kp->ks_type;
 705 
 706                 list_create(&ksi->ks_nvlist, sizeof (ks_nvpair_t),
 707                     offsetof(ks_nvpair_t, nv_next));
 708 
 709                 SAVE_HRTIME_X(ksi, "crtime", kp->ks_crtime);
 710                 SAVE_HRTIME_X(ksi, "snaptime", kp->ks_snaptime);
 711                 if (g_pflg) {
 712                         SAVE_STRING_X(ksi, "class", kp->ks_class);
 713                 }
 714 
 715                 /* Insert this instance into a sorted list */
 716                 tmp = list_head(&instances_list);
 717                 while (tmp != NULL && compare_instances(ksi, tmp) > 0)
 718                         tmp = list_next(&instances_list, tmp);
 719 
 720                 list_insert_before(&instances_list, tmp, ksi);
 721 
 722                 /* Read the actual statistics */
 723                 id = kstat_read(kc, kp, NULL);
 724                 if (id == -1) {
 725 #ifdef REPORT_UNKNOWN
 726                         perror("kstat_read");
 727 #endif
 728                         continue;
 729                 }
 730 
 731                 switch (kp->ks_type) {
 732                 case KSTAT_TYPE_RAW:
 733                         save_raw(kp, ksi);
 734                         break;
 735                 case KSTAT_TYPE_NAMED:
 736                         save_named(kp, ksi);
 737                         break;
 738                 case KSTAT_TYPE_INTR:
 739                         save_intr(kp, ksi);
 740                         break;
 741                 case KSTAT_TYPE_IO:
 742                         save_io(kp, ksi);
 743                         break;
 744                 case KSTAT_TYPE_TIMER:
 745                         save_timer(kp, ksi);
 746                         break;
 747                 default:
 748                         assert(B_FALSE); /* Invalid type */
 749                         break;
 750                 }
 751         }
 752 }
 753 
 754 /*
 755  * Print the value of a name-value pair.
 756  */
 757 static void
 758 ks_value_print(ks_nvpair_t *nvpair)
 759 {
 760         char dstr[64];
 761         char zstr[8];
 762         time_t t;
 763 
 764         switch (nvpair->data_type) {
 765         case KSTAT_DATA_CHAR:
 766                 (void) fprintf(stdout, "%s", nvpair->value.c);
 767                 break;
 768         case KSTAT_DATA_INT32:
 769                 (void) fprintf(stdout, "%d", nvpair->value.i32);
 770                 break;
 771         case KSTAT_DATA_UINT32:
 772                 (void) fprintf(stdout, "%u", nvpair->value.ui32);
 773                 break;
 774         case KSTAT_DATA_INT64:
 775                 (void) fprintf(stdout, "%lld", nvpair->value.i64);
 776                 break;
 777         case KSTAT_DATA_UINT64:
 778                 (void) fprintf(stdout, "%llu", nvpair->value.ui64);
 779                 break;
 780         case KSTAT_DATA_STRING:
 781                 (void) fprintf(stdout, "%s", KSTAT_NAMED_STR_PTR(nvpair));
 782                 break;
 783         case KSTAT_DATA_TIME:
 784                 switch (g_hrtime_fmt) {
 785                 case KS_HRFMT_UNIX:
 786                         (void) fprintf(stdout, "%.9f",
 787                             (nvpair->value.t + hrtime_origin) / 1000000000.0);
 788                         break;
 789                 case KS_HRFMT_NANO:
 790                         (void) fprintf(stdout, "%llu",
 791                             (nvpair->value.t + hrtime_origin));
 792                         break;
 793                 case KS_HRFMT_DATE:
 794                         t = (time_t)
 795                             ((nvpair->value.t + hrtime_origin) / 1000000000);
 796                         (void) strftime(dstr,
 797                             sizeof (dstr), "%+", localtime(&t));
 798                         (void) fprintf(stdout, "%s", dstr);
 799                         break;
 800                 case KS_HRFMT_ZISO:
 801                         t = (time_t)
 802                             ((nvpair->value.t + hrtime_origin) / 1000000000);
 803                         (void) strftime(dstr, sizeof (dstr), "%FT%T",
 804                             gmtime(&t));
 805                         (void) fprintf(stdout, "%s.%06uZ", dstr,
 806                             (unsigned)(nvpair->value.t % 1000000000)/1000);
 807                         break;
 808                 case KS_HRFMT_ISO:
 809                         t = (time_t)
 810                             ((nvpair->value.t + hrtime_origin) / 1000000000);
 811                         (void) strftime(dstr, sizeof (dstr), "%FT%T",
 812                             localtime(&t));
 813                         (void) strftime(zstr, sizeof (zstr), "%z",
 814                             localtime(&t));
 815                         (void) fprintf(stdout, "%s.%06u%s", dstr,
 816                             (unsigned)(nvpair->value.t % 1000000000)/1000,
 817                             zstr);
 818                         break;
 819                 case KS_HRFMT_DEFAULT:
 820                 case KS_HRFMT_BOOT:
 821                         if (nvpair->value.ui64 == 0)
 822                                 (void) fprintf(stdout, "0");
 823                         else
 824                                 (void) fprintf(stdout, "%.9f",
 825                                     nvpair->value.ui64 / 1000000000.0);
 826                         break;
 827                 default:
 828                         abort();
 829                 }
 830                 break;
 831         default:
 832                 assert(B_FALSE);
 833         }
 834 }
 835 
 836 /*
 837  * Print a single instance.
 838  */
 839 static void
 840 ks_instance_print(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 841 {
 842         if (g_headerflg) {
 843                 if (!g_pflg) {
 844                         (void) fprintf(stdout, DFLT_FMT,
 845                             ksi->ks_module, ksi->ks_instance,
 846                             ksi->ks_name, ksi->ks_class);
 847                 }
 848                 g_headerflg = B_FALSE;
 849         }
 850 
 851         if (g_pflg) {
 852                 (void) fprintf(stdout, KS_PFMT,
 853                     ksi->ks_module, ksi->ks_instance,
 854                     ksi->ks_name, nvpair->name);
 855                 if (!g_lflg) {
 856                         (void) putchar(g_cflg ? ':': '\t');
 857                         ks_value_print(nvpair);
 858                 }
 859         } else {
 860                 (void) fprintf(stdout, KS_DFMT, nvpair->name);
 861                 ks_value_print(nvpair);
 862         }
 863 
 864         (void) putchar('\n');
 865 }
 866 
 867 /*
 868  * Print a single instance in JSON format.
 869  */
 870 static void
 871 ks_instance_print_json(ks_instance_t *ksi, ks_nvpair_t *nvpair)
 872 {
 873         if (g_headerflg) {
 874                 (void) fprintf(stdout, JSON_FMT,
 875                     ksi->ks_module, ksi->ks_instance,
 876                     ksi->ks_name, ksi->ks_class,
 877                     ksi->ks_type);
 878 
 879                 if (ksi->ks_snaptime == 0)
 880                         (void) fprintf(stdout, "\t\"snaptime\": 0,\n");
 881                 else
 882                         (void) fprintf(stdout, "\t\"snaptime\": %.9f,\n",
 883                             ksi->ks_snaptime / 1000000000.0);
 884 
 885                 (void) fprintf(stdout, "\t\"data\": {\n");
 886 
 887                 g_headerflg = B_FALSE;
 888         }
 889 
 890         (void) fprintf(stdout, KS_JFMT, nvpair->name);
 891         if (nvpair->data_type == KSTAT_DATA_STRING) {
 892                 (void) putchar('\"');
 893                 ks_value_print(nvpair);
 894                 (void) putchar('\"');
 895         } else if (nvpair->data_type == KSTAT_DATA_TIME) {
 896                 switch (g_hrtime_fmt) {
 897                 case KS_HRFMT_ISO:
 898                 case KS_HRFMT_ZISO:
 899                 case KS_HRFMT_DATE:
 900                         (void) putchar('\"');
 901                         ks_value_print(nvpair);
 902                         (void) putchar('\"');
 903                         break;
 904                 default:
 905                         ks_value_print(nvpair);
 906                 }
 907         } else {
 908                 ks_value_print(nvpair);
 909         }
 910         if (nvpair != list_tail(&ksi->ks_nvlist))
 911                 (void) putchar(',');
 912 
 913         (void) putchar('\n');
 914 }
 915 
 916 /*
 917  * Print all instances.
 918  */
 919 static void
 920 ks_instances_print(void)
 921 {
 922         ks_selector_t   *selector;
 923         ks_instance_t   *ksi, *ktmp;
 924         ks_nvpair_t     *nvpair, *ntmp;
 925         void            (*ks_print_fn)(ks_instance_t *, ks_nvpair_t *);
 926         char            *ks_number;
 927 
 928         if (g_timestamp_fmt != NODATE)
 929                 print_timestamp(g_timestamp_fmt);
 930 
 931         if (g_jflg) {
 932                 ks_print_fn = &ks_instance_print_json;
 933                 (void) putchar('[');
 934         } else {
 935                 ks_print_fn = &ks_instance_print;
 936         }
 937 
 938         /* Iterate over each selector */
 939         selector = list_head(&selector_list);
 940         while (selector != NULL) {
 941 
 942                 /* Iterate over each instance */
 943                 for (ksi = list_head(&instances_list); ksi != NULL;
 944                     ksi = list_next(&instances_list, ksi)) {
 945 
 946                         (void) asprintf(&ks_number, "%d", ksi->ks_instance);
 947                         if (!(ks_match(ksi->ks_module, &selector->ks_module) &&
 948                             ks_match(ksi->ks_name, &selector->ks_name) &&
 949                             ks_match(ks_number, &selector->ks_instance) &&
 950                             ks_match(ksi->ks_class, &g_ks_class))) {
 951                                 free(ks_number);
 952                                 continue;
 953                         }
 954 
 955                         free(ks_number);
 956 
 957                         /* Finally iterate over each statistic */
 958                         g_headerflg = B_TRUE;
 959                         for (nvpair = list_head(&ksi->ks_nvlist);
 960                             nvpair != NULL;
 961                             nvpair = list_next(&ksi->ks_nvlist, nvpair)) {
 962                                 if (!ks_match(nvpair->name,
 963                                     &selector->ks_statistic))
 964                                         continue;
 965 
 966                                 g_matched = 0;
 967                                 if (!g_qflg)
 968                                         (*ks_print_fn)(ksi, nvpair);
 969                         }
 970 
 971                         if (!g_headerflg) {
 972                                 if (g_jflg) {
 973                                         (void) fprintf(stdout, "\t}\n}");
 974                                         if (ksi != list_tail(&instances_list))
 975                                                 (void) putchar(',');
 976                                 } else if (!g_pflg) {
 977                                         (void) putchar('\n');
 978                                 }
 979                         }
 980                 }
 981 
 982                 selector = list_next(&selector_list, selector);
 983         }
 984 
 985         if (g_jflg)
 986                 (void) fprintf(stdout, "]\n");
 987 
 988         (void) fflush(stdout);
 989 
 990         /* Free the instances list */
 991         ksi = list_head(&instances_list);
 992         while (ksi != NULL) {
 993                 nvpair = list_head(&ksi->ks_nvlist);
 994                 while (nvpair != NULL) {
 995                         ntmp = nvpair;
 996                         nvpair = list_next(&ksi->ks_nvlist, nvpair);
 997                         list_remove(&ksi->ks_nvlist, ntmp);
 998                         if (ntmp->data_type == KSTAT_DATA_STRING)
 999                                 free(ntmp->value.str.addr.ptr);
1000                         free(ntmp);
1001                 }
1002 
1003                 ktmp = ksi;
1004                 ksi = list_next(&instances_list, ksi);
1005                 list_remove(&instances_list, ktmp);
1006                 list_destroy(&ktmp->ks_nvlist);
1007                 free(ktmp);
1008         }
1009 }
1010 
1011 static void
1012 save_cpu_stat(kstat_t *kp, ks_instance_t *ksi)
1013 {
1014         cpu_stat_t      *stat;
1015         cpu_sysinfo_t   *sysinfo;
1016         cpu_syswait_t   *syswait;
1017         cpu_vminfo_t    *vminfo;
1018 
1019         stat = (cpu_stat_t *)(kp->ks_data);
1020         sysinfo = &stat->cpu_sysinfo;
1021         syswait = &stat->cpu_syswait;
1022         vminfo  = &stat->cpu_vminfo;
1023 
1024         SAVE_UINT32_X(ksi, "idle", sysinfo->cpu[CPU_IDLE]);
1025         SAVE_UINT32_X(ksi, "user", sysinfo->cpu[CPU_USER]);
1026         SAVE_UINT32_X(ksi, "kernel", sysinfo->cpu[CPU_KERNEL]);
1027         SAVE_UINT32_X(ksi, "wait", sysinfo->cpu[CPU_WAIT]);
1028         SAVE_UINT32_X(ksi, "wait_io", sysinfo->wait[W_IO]);
1029         SAVE_UINT32_X(ksi, "wait_swap", sysinfo->wait[W_SWAP]);
1030         SAVE_UINT32_X(ksi, "wait_pio", sysinfo->wait[W_PIO]);
1031         SAVE_UINT32(ksi, sysinfo, bread);
1032         SAVE_UINT32(ksi, sysinfo, bwrite);
1033         SAVE_UINT32(ksi, sysinfo, lread);
1034         SAVE_UINT32(ksi, sysinfo, lwrite);
1035         SAVE_UINT32(ksi, sysinfo, phread);
1036         SAVE_UINT32(ksi, sysinfo, phwrite);
1037         SAVE_UINT32(ksi, sysinfo, pswitch);
1038         SAVE_UINT32(ksi, sysinfo, trap);
1039         SAVE_UINT32(ksi, sysinfo, intr);
1040         SAVE_UINT32(ksi, sysinfo, syscall);
1041         SAVE_UINT32(ksi, sysinfo, sysread);
1042         SAVE_UINT32(ksi, sysinfo, syswrite);
1043         SAVE_UINT32(ksi, sysinfo, sysfork);
1044         SAVE_UINT32(ksi, sysinfo, sysvfork);
1045         SAVE_UINT32(ksi, sysinfo, sysexec);
1046         SAVE_UINT32(ksi, sysinfo, readch);
1047         SAVE_UINT32(ksi, sysinfo, writech);
1048         SAVE_UINT32(ksi, sysinfo, rcvint);
1049         SAVE_UINT32(ksi, sysinfo, xmtint);
1050         SAVE_UINT32(ksi, sysinfo, mdmint);
1051         SAVE_UINT32(ksi, sysinfo, rawch);
1052         SAVE_UINT32(ksi, sysinfo, canch);
1053         SAVE_UINT32(ksi, sysinfo, outch);
1054         SAVE_UINT32(ksi, sysinfo, msg);
1055         SAVE_UINT32(ksi, sysinfo, sema);
1056         SAVE_UINT32(ksi, sysinfo, namei);
1057         SAVE_UINT32(ksi, sysinfo, ufsiget);
1058         SAVE_UINT32(ksi, sysinfo, ufsdirblk);
1059         SAVE_UINT32(ksi, sysinfo, ufsipage);
1060         SAVE_UINT32(ksi, sysinfo, ufsinopage);
1061         SAVE_UINT32(ksi, sysinfo, inodeovf);
1062         SAVE_UINT32(ksi, sysinfo, fileovf);
1063         SAVE_UINT32(ksi, sysinfo, procovf);
1064         SAVE_UINT32(ksi, sysinfo, intrthread);
1065         SAVE_UINT32(ksi, sysinfo, intrblk);
1066         SAVE_UINT32(ksi, sysinfo, idlethread);
1067         SAVE_UINT32(ksi, sysinfo, inv_swtch);
1068         SAVE_UINT32(ksi, sysinfo, nthreads);
1069         SAVE_UINT32(ksi, sysinfo, cpumigrate);
1070         SAVE_UINT32(ksi, sysinfo, xcalls);
1071         SAVE_UINT32(ksi, sysinfo, mutex_adenters);
1072         SAVE_UINT32(ksi, sysinfo, rw_rdfails);
1073         SAVE_UINT32(ksi, sysinfo, rw_wrfails);
1074         SAVE_UINT32(ksi, sysinfo, modload);
1075         SAVE_UINT32(ksi, sysinfo, modunload);
1076         SAVE_UINT32(ksi, sysinfo, bawrite);
1077 #ifdef  STATISTICS      /* see header file */
1078         SAVE_UINT32(ksi, sysinfo, rw_enters);
1079         SAVE_UINT32(ksi, sysinfo, win_uo_cnt);
1080         SAVE_UINT32(ksi, sysinfo, win_uu_cnt);
1081         SAVE_UINT32(ksi, sysinfo, win_so_cnt);
1082         SAVE_UINT32(ksi, sysinfo, win_su_cnt);
1083         SAVE_UINT32(ksi, sysinfo, win_suo_cnt);
1084 #endif
1085 
1086         SAVE_INT32(ksi, syswait, iowait);
1087         SAVE_INT32(ksi, syswait, swap);
1088         SAVE_INT32(ksi, syswait, physio);
1089 
1090         SAVE_UINT32(ksi, vminfo, pgrec);
1091         SAVE_UINT32(ksi, vminfo, pgfrec);
1092         SAVE_UINT32(ksi, vminfo, pgin);
1093         SAVE_UINT32(ksi, vminfo, pgpgin);
1094         SAVE_UINT32(ksi, vminfo, pgout);
1095         SAVE_UINT32(ksi, vminfo, pgpgout);
1096         SAVE_UINT32(ksi, vminfo, swapin);
1097         SAVE_UINT32(ksi, vminfo, pgswapin);
1098         SAVE_UINT32(ksi, vminfo, swapout);
1099         SAVE_UINT32(ksi, vminfo, pgswapout);
1100         SAVE_UINT32(ksi, vminfo, zfod);
1101         SAVE_UINT32(ksi, vminfo, dfree);
1102         SAVE_UINT32(ksi, vminfo, scan);
1103         SAVE_UINT32(ksi, vminfo, rev);
1104         SAVE_UINT32(ksi, vminfo, hat_fault);
1105         SAVE_UINT32(ksi, vminfo, as_fault);
1106         SAVE_UINT32(ksi, vminfo, maj_fault);
1107         SAVE_UINT32(ksi, vminfo, cow_fault);
1108         SAVE_UINT32(ksi, vminfo, prot_fault);
1109         SAVE_UINT32(ksi, vminfo, softlock);
1110         SAVE_UINT32(ksi, vminfo, kernel_asflt);
1111         SAVE_UINT32(ksi, vminfo, pgrrun);
1112         SAVE_UINT32(ksi, vminfo, execpgin);
1113         SAVE_UINT32(ksi, vminfo, execpgout);
1114         SAVE_UINT32(ksi, vminfo, execfree);
1115         SAVE_UINT32(ksi, vminfo, anonpgin);
1116         SAVE_UINT32(ksi, vminfo, anonpgout);
1117         SAVE_UINT32(ksi, vminfo, anonfree);
1118         SAVE_UINT32(ksi, vminfo, fspgin);
1119         SAVE_UINT32(ksi, vminfo, fspgout);
1120         SAVE_UINT32(ksi, vminfo, fsfree);
1121 }
1122 
1123 static void
1124 save_var(kstat_t *kp, ks_instance_t *ksi)
1125 {
1126         struct var      *var = (struct var *)(kp->ks_data);
1127 
1128         assert(kp->ks_data_size == sizeof (struct var));
1129 
1130         SAVE_INT32(ksi, var, v_buf);
1131         SAVE_INT32(ksi, var, v_call);
1132         SAVE_INT32(ksi, var, v_proc);
1133         SAVE_INT32(ksi, var, v_maxupttl);
1134         SAVE_INT32(ksi, var, v_nglobpris);
1135         SAVE_INT32(ksi, var, v_maxsyspri);
1136         SAVE_INT32(ksi, var, v_clist);
1137         SAVE_INT32(ksi, var, v_maxup);
1138         SAVE_INT32(ksi, var, v_hbuf);
1139         SAVE_INT32(ksi, var, v_hmask);
1140         SAVE_INT32(ksi, var, v_pbuf);
1141         SAVE_INT32(ksi, var, v_sptmap);
1142         SAVE_INT32(ksi, var, v_maxpmem);
1143         SAVE_INT32(ksi, var, v_autoup);
1144         SAVE_INT32(ksi, var, v_bufhwm);
1145 }
1146 
1147 static void
1148 save_ncstats(kstat_t *kp, ks_instance_t *ksi)
1149 {
1150         struct ncstats  *ncstats = (struct ncstats *)(kp->ks_data);
1151 
1152         assert(kp->ks_data_size == sizeof (struct ncstats));
1153 
1154         SAVE_INT32(ksi, ncstats, hits);
1155         SAVE_INT32(ksi, ncstats, misses);
1156         SAVE_INT32(ksi, ncstats, enters);
1157         SAVE_INT32(ksi, ncstats, dbl_enters);
1158         SAVE_INT32(ksi, ncstats, long_enter);
1159         SAVE_INT32(ksi, ncstats, long_look);
1160         SAVE_INT32(ksi, ncstats, move_to_front);
1161         SAVE_INT32(ksi, ncstats, purges);
1162 }
1163 
1164 static void
1165 save_sysinfo(kstat_t *kp, ks_instance_t *ksi)
1166 {
1167         sysinfo_t       *sysinfo = (sysinfo_t *)(kp->ks_data);
1168 
1169         assert(kp->ks_data_size == sizeof (sysinfo_t));
1170 
1171         SAVE_UINT32(ksi, sysinfo, updates);
1172         SAVE_UINT32(ksi, sysinfo, runque);
1173         SAVE_UINT32(ksi, sysinfo, runocc);
1174         SAVE_UINT32(ksi, sysinfo, swpque);
1175         SAVE_UINT32(ksi, sysinfo, swpocc);
1176         SAVE_UINT32(ksi, sysinfo, waiting);
1177 }
1178 
1179 static void
1180 save_vminfo(kstat_t *kp, ks_instance_t *ksi)
1181 {
1182         vminfo_t        *vminfo = (vminfo_t *)(kp->ks_data);
1183 
1184         assert(kp->ks_data_size == sizeof (vminfo_t));
1185 
1186         SAVE_UINT64(ksi, vminfo, freemem);
1187         SAVE_UINT64(ksi, vminfo, swap_resv);
1188         SAVE_UINT64(ksi, vminfo, swap_alloc);
1189         SAVE_UINT64(ksi, vminfo, swap_avail);
1190         SAVE_UINT64(ksi, vminfo, swap_free);
1191         SAVE_UINT64(ksi, vminfo, updates);
1192 }
1193 
1194 static void
1195 save_nfs(kstat_t *kp, ks_instance_t *ksi)
1196 {
1197         struct mntinfo_kstat *mntinfo = (struct mntinfo_kstat *)(kp->ks_data);
1198 
1199         assert(kp->ks_data_size == sizeof (struct mntinfo_kstat));
1200 
1201         SAVE_STRING(ksi, mntinfo, mik_proto);
1202         SAVE_UINT32(ksi, mntinfo, mik_vers);
1203         SAVE_UINT32(ksi, mntinfo, mik_flags);
1204         SAVE_UINT32(ksi, mntinfo, mik_secmod);
1205         SAVE_UINT32(ksi, mntinfo, mik_curread);
1206         SAVE_UINT32(ksi, mntinfo, mik_curwrite);
1207         SAVE_INT32(ksi, mntinfo, mik_timeo);
1208         SAVE_INT32(ksi, mntinfo, mik_retrans);
1209         SAVE_UINT32(ksi, mntinfo, mik_acregmin);
1210         SAVE_UINT32(ksi, mntinfo, mik_acregmax);
1211         SAVE_UINT32(ksi, mntinfo, mik_acdirmin);
1212         SAVE_UINT32(ksi, mntinfo, mik_acdirmax);
1213         SAVE_UINT32_X(ksi, "lookup_srtt", mntinfo->mik_timers[0].srtt);
1214         SAVE_UINT32_X(ksi, "lookup_deviate", mntinfo->mik_timers[0].deviate);
1215         SAVE_UINT32_X(ksi, "lookup_rtxcur", mntinfo->mik_timers[0].rtxcur);
1216         SAVE_UINT32_X(ksi, "read_srtt", mntinfo->mik_timers[1].srtt);
1217         SAVE_UINT32_X(ksi, "read_deviate", mntinfo->mik_timers[1].deviate);
1218         SAVE_UINT32_X(ksi, "read_rtxcur", mntinfo->mik_timers[1].rtxcur);
1219         SAVE_UINT32_X(ksi, "write_srtt", mntinfo->mik_timers[2].srtt);
1220         SAVE_UINT32_X(ksi, "write_deviate", mntinfo->mik_timers[2].deviate);
1221         SAVE_UINT32_X(ksi, "write_rtxcur", mntinfo->mik_timers[2].rtxcur);
1222         SAVE_UINT32(ksi, mntinfo, mik_noresponse);
1223         SAVE_UINT32(ksi, mntinfo, mik_failover);
1224         SAVE_UINT32(ksi, mntinfo, mik_remap);
1225         SAVE_STRING(ksi, mntinfo, mik_curserver);
1226 }
1227 
1228 #ifdef __sparc
1229 static void
1230 save_sfmmu_global_stat(kstat_t *kp, ks_instance_t *ksi)
1231 {
1232         struct sfmmu_global_stat *sfmmug =
1233             (struct sfmmu_global_stat *)(kp->ks_data);
1234 
1235         assert(kp->ks_data_size == sizeof (struct sfmmu_global_stat));
1236 
1237         SAVE_INT32(ksi, sfmmug, sf_tsb_exceptions);
1238         SAVE_INT32(ksi, sfmmug, sf_tsb_raise_exception);
1239         SAVE_INT32(ksi, sfmmug, sf_pagefaults);
1240         SAVE_INT32(ksi, sfmmug, sf_uhash_searches);
1241         SAVE_INT32(ksi, sfmmug, sf_uhash_links);
1242         SAVE_INT32(ksi, sfmmug, sf_khash_searches);
1243         SAVE_INT32(ksi, sfmmug, sf_khash_links);
1244         SAVE_INT32(ksi, sfmmug, sf_swapout);
1245         SAVE_INT32(ksi, sfmmug, sf_tsb_alloc);
1246         SAVE_INT32(ksi, sfmmug, sf_tsb_allocfail);
1247         SAVE_INT32(ksi, sfmmug, sf_tsb_sectsb_create);
1248         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_alloc);
1249         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_alloc);
1250         SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_allocfail);
1251         SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_allocfail);
1252         SAVE_INT32(ksi, sfmmug, sf_tteload8k);
1253         SAVE_INT32(ksi, sfmmug, sf_tteload64k);
1254         SAVE_INT32(ksi, sfmmug, sf_tteload512k);
1255         SAVE_INT32(ksi, sfmmug, sf_tteload4m);
1256         SAVE_INT32(ksi, sfmmug, sf_tteload32m);
1257         SAVE_INT32(ksi, sfmmug, sf_tteload256m);
1258         SAVE_INT32(ksi, sfmmug, sf_tsb_load8k);
1259         SAVE_INT32(ksi, sfmmug, sf_tsb_load4m);
1260         SAVE_INT32(ksi, sfmmug, sf_hblk_hit);
1261         SAVE_INT32(ksi, sfmmug, sf_hblk8_ncreate);
1262         SAVE_INT32(ksi, sfmmug, sf_hblk8_nalloc);
1263         SAVE_INT32(ksi, sfmmug, sf_hblk1_ncreate);
1264         SAVE_INT32(ksi, sfmmug, sf_hblk1_nalloc);
1265         SAVE_INT32(ksi, sfmmug, sf_hblk_slab_cnt);
1266         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_cnt);
1267         SAVE_INT32(ksi, sfmmug, sf_hblk_recurse_cnt);
1268         SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_hit);
1269         SAVE_INT32(ksi, sfmmug, sf_get_free_success);
1270         SAVE_INT32(ksi, sfmmug, sf_get_free_throttle);
1271         SAVE_INT32(ksi, sfmmug, sf_get_free_fail);
1272         SAVE_INT32(ksi, sfmmug, sf_put_free_success);
1273         SAVE_INT32(ksi, sfmmug, sf_put_free_fail);
1274         SAVE_INT32(ksi, sfmmug, sf_pgcolor_conflict);
1275         SAVE_INT32(ksi, sfmmug, sf_uncache_conflict);
1276         SAVE_INT32(ksi, sfmmug, sf_unload_conflict);
1277         SAVE_INT32(ksi, sfmmug, sf_ism_uncache);
1278         SAVE_INT32(ksi, sfmmug, sf_ism_recache);
1279         SAVE_INT32(ksi, sfmmug, sf_recache);
1280         SAVE_INT32(ksi, sfmmug, sf_steal_count);
1281         SAVE_INT32(ksi, sfmmug, sf_pagesync);
1282         SAVE_INT32(ksi, sfmmug, sf_clrwrt);
1283         SAVE_INT32(ksi, sfmmug, sf_pagesync_invalid);
1284         SAVE_INT32(ksi, sfmmug, sf_kernel_xcalls);
1285         SAVE_INT32(ksi, sfmmug, sf_user_xcalls);
1286         SAVE_INT32(ksi, sfmmug, sf_tsb_grow);
1287         SAVE_INT32(ksi, sfmmug, sf_tsb_shrink);
1288         SAVE_INT32(ksi, sfmmug, sf_tsb_resize_failures);
1289         SAVE_INT32(ksi, sfmmug, sf_tsb_reloc);
1290         SAVE_INT32(ksi, sfmmug, sf_user_vtop);
1291         SAVE_INT32(ksi, sfmmug, sf_ctx_inv);
1292         SAVE_INT32(ksi, sfmmug, sf_tlb_reprog_pgsz);
1293         SAVE_INT32(ksi, sfmmug, sf_region_remap_demap);
1294         SAVE_INT32(ksi, sfmmug, sf_create_scd);
1295         SAVE_INT32(ksi, sfmmug, sf_join_scd);
1296         SAVE_INT32(ksi, sfmmug, sf_leave_scd);
1297         SAVE_INT32(ksi, sfmmug, sf_destroy_scd);
1298 }
1299 #endif
1300 
1301 #ifdef __sparc
1302 static void
1303 save_sfmmu_tsbsize_stat(kstat_t *kp, ks_instance_t *ksi)
1304 {
1305         struct sfmmu_tsbsize_stat *sfmmut;
1306 
1307         assert(kp->ks_data_size == sizeof (struct sfmmu_tsbsize_stat));
1308         sfmmut = (struct sfmmu_tsbsize_stat *)(kp->ks_data);
1309 
1310         SAVE_INT32(ksi, sfmmut, sf_tsbsz_8k);
1311         SAVE_INT32(ksi, sfmmut, sf_tsbsz_16k);
1312         SAVE_INT32(ksi, sfmmut, sf_tsbsz_32k);
1313         SAVE_INT32(ksi, sfmmut, sf_tsbsz_64k);
1314         SAVE_INT32(ksi, sfmmut, sf_tsbsz_128k);
1315         SAVE_INT32(ksi, sfmmut, sf_tsbsz_256k);
1316         SAVE_INT32(ksi, sfmmut, sf_tsbsz_512k);
1317         SAVE_INT32(ksi, sfmmut, sf_tsbsz_1m);
1318         SAVE_INT32(ksi, sfmmut, sf_tsbsz_2m);
1319         SAVE_INT32(ksi, sfmmut, sf_tsbsz_4m);
1320 }
1321 #endif
1322 
1323 #ifdef __sparc
1324 static void
1325 save_simmstat(kstat_t *kp, ks_instance_t *ksi)
1326 {
1327         uchar_t *simmstat;
1328         char    *simm_buf;
1329         char    *list = NULL;
1330         int     i;
1331 
1332         assert(kp->ks_data_size == sizeof (uchar_t) * SIMM_COUNT);
1333 
1334         for (i = 0, simmstat = (uchar_t *)(kp->ks_data); i < SIMM_COUNT - 1;
1335             i++, simmstat++) {
1336                 if (list == NULL) {
1337                         (void) asprintf(&simm_buf, "%d,", *simmstat);
1338                 } else {
1339                         (void) asprintf(&simm_buf, "%s%d,", list, *simmstat);
1340                         free(list);
1341                 }
1342                 list = simm_buf;
1343         }
1344 
1345         (void) asprintf(&simm_buf, "%s%d", list, *simmstat);
1346         SAVE_STRING_X(ksi, "status", simm_buf);
1347         free(list);
1348         free(simm_buf);
1349 }
1350 #endif
1351 
1352 #ifdef __sparc
1353 /*
1354  * Helper function for save_temperature().
1355  */
1356 static char *
1357 short_array_to_string(short *shortp, int len)
1358 {
1359         char    *list = NULL;
1360         char    *list_buf;
1361 
1362         for (; len > 1; len--, shortp++) {
1363                 if (list == NULL) {
1364                         (void) asprintf(&list_buf, "%hd,", *shortp);
1365                 } else {
1366                         (void) asprintf(&list_buf, "%s%hd,", list, *shortp);
1367                         free(list);
1368                 }
1369                 list = list_buf;
1370         }
1371 
1372         (void) asprintf(&list_buf, "%s%hd", list, *shortp);
1373         free(list);
1374         return (list_buf);
1375 }
1376 
1377 static void
1378 save_temperature(kstat_t *kp, ks_instance_t *ksi)
1379 {
1380         struct temp_stats *temps = (struct temp_stats *)(kp->ks_data);
1381         char    *buf;
1382 
1383         assert(kp->ks_data_size == sizeof (struct temp_stats));
1384 
1385         SAVE_UINT32(ksi, temps, index);
1386 
1387         buf = short_array_to_string(temps->l1, L1_SZ);
1388         SAVE_STRING_X(ksi, "l1", buf);
1389         free(buf);
1390 
1391         buf = short_array_to_string(temps->l2, L2_SZ);
1392         SAVE_STRING_X(ksi, "l2", buf);
1393         free(buf);
1394 
1395         buf = short_array_to_string(temps->l3, L3_SZ);
1396         SAVE_STRING_X(ksi, "l3", buf);
1397         free(buf);
1398 
1399         buf = short_array_to_string(temps->l4, L4_SZ);
1400         SAVE_STRING_X(ksi, "l4", buf);
1401         free(buf);
1402 
1403         buf = short_array_to_string(temps->l5, L5_SZ);
1404         SAVE_STRING_X(ksi, "l5", buf);
1405         free(buf);
1406 
1407         SAVE_INT32(ksi, temps, max);
1408         SAVE_INT32(ksi, temps, min);
1409         SAVE_INT32(ksi, temps, state);
1410         SAVE_INT32(ksi, temps, temp_cnt);
1411         SAVE_INT32(ksi, temps, shutdown_cnt);
1412         SAVE_INT32(ksi, temps, version);
1413         SAVE_INT32(ksi, temps, trend);
1414         SAVE_INT32(ksi, temps, override);
1415 }
1416 #endif
1417 
1418 #ifdef __sparc
1419 static void
1420 save_temp_over(kstat_t *kp, ks_instance_t *ksi)
1421 {
1422         short   *sh = (short *)(kp->ks_data);
1423         char    *value;
1424 
1425         assert(kp->ks_data_size == sizeof (short));
1426 
1427         (void) asprintf(&value, "%hu", *sh);
1428         SAVE_STRING_X(ksi, "override", value);
1429         free(value);
1430 }
1431 #endif
1432 
1433 #ifdef __sparc
1434 static void
1435 save_ps_shadow(kstat_t *kp, ks_instance_t *ksi)
1436 {
1437         uchar_t *uchar = (uchar_t *)(kp->ks_data);
1438 
1439         assert(kp->ks_data_size == SYS_PS_COUNT);
1440 
1441         SAVE_CHAR_X(ksi, "core_0", *uchar++);
1442         SAVE_CHAR_X(ksi, "core_1", *uchar++);
1443         SAVE_CHAR_X(ksi, "core_2", *uchar++);
1444         SAVE_CHAR_X(ksi, "core_3", *uchar++);
1445         SAVE_CHAR_X(ksi, "core_4", *uchar++);
1446         SAVE_CHAR_X(ksi, "core_5", *uchar++);
1447         SAVE_CHAR_X(ksi, "core_6", *uchar++);
1448         SAVE_CHAR_X(ksi, "core_7", *uchar++);
1449         SAVE_CHAR_X(ksi, "pps_0", *uchar++);
1450         SAVE_CHAR_X(ksi, "clk_33", *uchar++);
1451         SAVE_CHAR_X(ksi, "clk_50", *uchar++);
1452         SAVE_CHAR_X(ksi, "v5_p", *uchar++);
1453         SAVE_CHAR_X(ksi, "v12_p", *uchar++);
1454         SAVE_CHAR_X(ksi, "v5_aux", *uchar++);
1455         SAVE_CHAR_X(ksi, "v5_p_pch", *uchar++);
1456         SAVE_CHAR_X(ksi, "v12_p_pch", *uchar++);
1457         SAVE_CHAR_X(ksi, "v3_pch", *uchar++);
1458         SAVE_CHAR_X(ksi, "v5_pch", *uchar++);
1459         SAVE_CHAR_X(ksi, "p_fan", *uchar++);
1460 }
1461 #endif
1462 
1463 #ifdef __sparc
1464 static void
1465 save_fault_list(kstat_t *kp, ks_instance_t *ksi)
1466 {
1467         struct ft_list *fault;
1468         char    name[KSTAT_STRLEN + 7];
1469         int     i;
1470 
1471         for (i = 1, fault = (struct ft_list *)(kp->ks_data);
1472             i <= 999999 && i <= kp->ks_data_size / sizeof (struct ft_list);
1473             i++, fault++) {
1474                 (void) snprintf(name, sizeof (name), "unit_%d", i);
1475                 SAVE_INT32_X(ksi, name, fault->unit);
1476                 (void) snprintf(name, sizeof (name), "type_%d", i);
1477                 SAVE_INT32_X(ksi, name, fault->type);
1478                 (void) snprintf(name, sizeof (name), "fclass_%d", i);
1479                 SAVE_INT32_X(ksi, name, fault->fclass);
1480                 (void) snprintf(name, sizeof (name), "create_time_%d", i);
1481                 SAVE_HRTIME_X(ksi, name, fault->create_time);
1482                 (void) snprintf(name, sizeof (name), "msg_%d", i);
1483                 SAVE_STRING_X(ksi, name, fault->msg);
1484         }
1485 }
1486 #endif
1487 
1488 static void
1489 save_named(kstat_t *kp, ks_instance_t *ksi)
1490 {
1491         kstat_named_t *knp;
1492         int     n;
1493 
1494         for (n = kp->ks_ndata, knp = KSTAT_NAMED_PTR(kp); n > 0; n--, knp++) {
1495                 switch (knp->data_type) {
1496                 case KSTAT_DATA_CHAR:
1497                 case KSTAT_DATA_INT32:
1498                 case KSTAT_DATA_UINT32:
1499                 case KSTAT_DATA_INT64:
1500                 case KSTAT_DATA_UINT64:
1501                 case KSTAT_DATA_TIME:
1502                         nvpair_insert(ksi, knp->name,
1503                             (ks_value_t *)&knp->value, knp->data_type);
1504                         break;
1505                 case KSTAT_DATA_STRING:
1506                         SAVE_STRING_X(ksi, knp->name, KSTAT_NAMED_STR_PTR(knp));
1507                         break;
1508                 default:
1509                         assert(B_FALSE); /* Invalid data type */
1510                         break;
1511                 }
1512         }
1513 }
1514 
1515 static void
1516 save_intr(kstat_t *kp, ks_instance_t *ksi)
1517 {
1518         kstat_intr_t *intr = KSTAT_INTR_PTR(kp);
1519         char    *intr_names[] = {"hard", "soft", "watchdog", "spurious",
1520             "multiple_service"};
1521         int     n;
1522 
1523         for (n = 0; n < KSTAT_NUM_INTRS; n++)
1524                 SAVE_UINT32_X(ksi, intr_names[n], intr->intrs[n]);
1525 }
1526 
1527 static void
1528 save_io(kstat_t *kp, ks_instance_t *ksi)
1529 {
1530         kstat_io_t      *ksio = KSTAT_IO_PTR(kp);
1531 
1532         SAVE_UINT64(ksi, ksio, nread);
1533         SAVE_UINT64(ksi, ksio, nwritten);
1534         SAVE_UINT32(ksi, ksio, reads);
1535         SAVE_UINT32(ksi, ksio, writes);
1536         SAVE_UINT64(ksi, ksio, wtime);
1537         SAVE_UINT64(ksi, ksio, wlentime);
1538         SAVE_HRTIME(ksi, ksio, wlastupdate);
1539         SAVE_UINT64(ksi, ksio, rtime);
1540         SAVE_UINT64(ksi, ksio, rlentime);
1541         SAVE_HRTIME(ksi, ksio, rlastupdate);
1542         SAVE_UINT32(ksi, ksio, wcnt);
1543         SAVE_UINT32(ksi, ksio, rcnt);
1544 }
1545 
1546 static void
1547 save_timer(kstat_t *kp, ks_instance_t *ksi)
1548 {
1549         kstat_timer_t   *ktimer = KSTAT_TIMER_PTR(kp);
1550 
1551         SAVE_STRING(ksi, ktimer, name);
1552         SAVE_UINT64(ksi, ktimer, num_events);
1553         SAVE_UINT64(ksi, ktimer, elapsed_time);
1554         SAVE_UINT64(ksi, ktimer, min_time);
1555         SAVE_UINT64(ksi, ktimer, max_time);
1556         SAVE_HRTIME(ksi, ktimer, start_time);
1557         SAVE_HRTIME(ksi, ktimer, stop_time);
1558 }