9936 atomic ops in syscall_mstate() induce significant overhead
9942 zone secflags are not initialized correctly

*** 19,29 ****
   * CDDL HEADER END
   */
  /*
   * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
!  * Copyright 2012 Joyent, Inc.  All rights reserved.
   */
  
  #include <sys/types.h>
  #include <sys/param.h>
  #include <sys/systm.h>
--- 19,29 ----
   * CDDL HEADER END
   */
  /*
   * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
!  * Copyright (c) 2018, Joyent, Inc.
   */
  
  #include <sys/types.h>
  #include <sys/param.h>
  #include <sys/systm.h>
*** 414,433 ****
          while (newtime < 0) {
                  curtime = gethrtime_unscaled();
                  newtime = curtime - ms->ms_state_start;
          }
          *mstimep += newtime;
-         if (fromms == LMS_USER)
-                 atomic_add_64(&z->zone_utime, newtime);
-         else if (fromms == LMS_SYSTEM)
-                 atomic_add_64(&z->zone_stime, newtime);
          t->t_mstate = toms;
          ms->ms_state_start = curtime;
          ms->ms_prev = fromms;
          kpreempt_disable(); /* don't change CPU while changing CPU's state */
          cpu = CPU;
          ASSERT(cpu == t->t_cpu);
          if ((toms != LMS_USER) && (cpu->cpu_mstate != CMS_SYSTEM)) {
                  NEW_CPU_MSTATE(CMS_SYSTEM);
          } else if ((toms == LMS_USER) && (cpu->cpu_mstate != CMS_USER)) {
                  NEW_CPU_MSTATE(CMS_USER);
          }
--- 414,438 ----
          while (newtime < 0) {
                  curtime = gethrtime_unscaled();
                  newtime = curtime - ms->ms_state_start;
          }
          *mstimep += newtime;
          t->t_mstate = toms;
          ms->ms_state_start = curtime;
          ms->ms_prev = fromms;
          kpreempt_disable(); /* don't change CPU while changing CPU's state */
          cpu = CPU;
          ASSERT(cpu == t->t_cpu);
+ 
+         if (fromms == LMS_USER) {
+                 CPU_UARRAY_VAL(z->zone_ustate, cpu->cpu_id,
+                     ZONE_USTATE_UTIME) += newtime;
+         } else if (fromms == LMS_SYSTEM) {
+                 CPU_UARRAY_VAL(z->zone_ustate, cpu->cpu_id,
+                     ZONE_USTATE_STIME) += newtime;
+         }
+ 
          if ((toms != LMS_USER) && (cpu->cpu_mstate != CMS_SYSTEM)) {
                  NEW_CPU_MSTATE(CMS_SYSTEM);
          } else if ((toms == LMS_USER) && (cpu->cpu_mstate != CMS_USER)) {
                  NEW_CPU_MSTATE(CMS_USER);
          }
*** 651,673 ****
                  ms->ms_state_start = curtime;
          } while (atomic_cas_64((uint64_t *)mstimep, oldtime, newtime) !=
              oldtime);
  
          /*
-          * When the system boots the initial startup thread will have a
-          * ms_state_start of 0 which would add a huge system time to the global
-          * zone.  We want to skip aggregating that initial bit of work.
-          */
-         if (origstart != 0) {
-                 z = ttozone(t);
-                 if (state == LMS_USER)
-                         atomic_add_64(&z->zone_utime, ztime);
-                 else if (state == LMS_SYSTEM)
-                         atomic_add_64(&z->zone_stime, ztime);
-         }
- 
-         /*
           * Remember the previous running microstate.
           */
          if (state != LMS_SLEEP && state != LMS_STOPPED)
                  ms->ms_prev = state;
  
--- 656,665 ----
*** 674,684 ****
--- 666,694 ----
          /*
           * Switch CPU microstate if appropriate
           */
  
          kpreempt_disable(); /* MUST disable kpreempt before touching t->cpu */
+ 
          ASSERT(t->t_cpu == CPU);
+ 
+         /*
+          * When the system boots the initial startup thread will have a
+          * ms_state_start of 0 which would add a huge system time to the global
+          * zone.  We want to skip aggregating that initial bit of work.
+          */
+         if (origstart != 0) {
+                 z = ttozone(t);
+                 if (state == LMS_USER) {
+                         CPU_UARRAY_VAL(z->zone_ustate, t->t_cpu->cpu_id,
+                             ZONE_USTATE_UTIME) += ztime;
+                 } else if (state == LMS_SYSTEM) {
+                         CPU_UARRAY_VAL(z->zone_ustate, t->t_cpu->cpu_id,
+                             ZONE_USTATE_STIME) += ztime;
+                 }
+         }
+ 
          if (!CPU_ON_INTR(t->t_cpu) && curthread->t_intr == NULL) {
                  if (new_state == LMS_USER && t->t_cpu->cpu_mstate != CMS_USER)
                          new_cpu_mstate(CMS_USER, curtime);
                  else if (new_state != LMS_USER &&
                      t->t_cpu->cpu_mstate != CMS_SYSTEM)
*** 781,791 ****
           * Update the WAIT_CPU timer and per-cpu waitrq total.
           */
          z = ttozone(t);
          waittime = curtime - waitrq;
          ms->ms_acct[LMS_WAIT_CPU] += waittime;
!         atomic_add_64(&z->zone_wtime, waittime);
          CPU->cpu_waitrq += waittime;
          ms->ms_state_start = curtime;
  }
  
  /*
--- 791,807 ----
           * Update the WAIT_CPU timer and per-cpu waitrq total.
           */
          z = ttozone(t);
          waittime = curtime - waitrq;
          ms->ms_acct[LMS_WAIT_CPU] += waittime;
! 
!         /*
!          * We are in a disp context where we're not going to migrate CPUs.
!          */
!         CPU_UARRAY_VAL(z->zone_ustate, CPU->cpu_id,
!             ZONE_USTATE_WTIME) += waittime;
! 
          CPU->cpu_waitrq += waittime;
          ms->ms_state_start = curtime;
  }
  
  /*