9936 atomic ops in syscall_mstate() induce significant overhead
9942 zone secflags are not initialized correctly

@@ -18,11 +18,11 @@
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2015 Joyent, Inc. All rights reserved.
+ * Copyright 2018 Joyent, Inc.
  * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
  */
 
 #ifndef _SYS_ZONE_H

@@ -40,10 +40,11 @@
 #include <sys/uadmin.h>
 #include <sys/ksynch.h>
 #include <sys/socket_impl.h>
 #include <sys/secflags.h>
 #include <netinet/in.h>
+#include <sys/cpu_uarray.h>
 
 #ifdef  __cplusplus
 extern "C" {
 #endif
 

@@ -328,10 +329,19 @@
 #include <sys/list.h>
 #include <sys/loadavg.h>
 
 #define GLOBAL_ZONEUNIQID       0       /* uniqid of the global zone */
 
+/*
+ * Indexes into ->zone_ustate array, summing the micro state of all threads in a
+ * particular zone.
+ */
+#define ZONE_USTATE_STIME (0)
+#define ZONE_USTATE_UTIME (1)
+#define ZONE_USTATE_WTIME (2)
+#define ZONE_USTATE_MAX (3)
+
 struct pool;
 struct brand;
 
 /*
  * Each of these constants identifies a kernel subsystem that acquires and

@@ -582,26 +592,16 @@
 
         psecflags_t     zone_secflags; /* default zone security-flags */
 
         /*
          * Misc. kstats and counters for zone cpu-usage aggregation.
-         * The zone_Xtime values are the sum of the micro-state accounting
-         * values for all threads that are running or have run in the zone.
-         * This is tracked in msacct.c as threads change state.
-         * The zone_stime is the sum of the LMS_SYSTEM times.
-         * The zone_utime is the sum of the LMS_USER times.
-         * The zone_wtime is the sum of the LMS_WAIT_CPU times.
-         * As with per-thread micro-state accounting values, these values are
-         * not scaled to nanosecs.  The scaling is done by the
-         * zone_misc_kstat_update function when kstats are requested.
          */
         kmutex_t        zone_misc_lock;         /* protects misc statistics */
         kstat_t         *zone_misc_ksp;
         zone_misc_kstat_t *zone_misc_stats;
-        uint64_t        zone_stime;             /* total system time */
-        uint64_t        zone_utime;             /* total user time */
-        uint64_t        zone_wtime;             /* total time waiting in runq */
+        /* Accumulated microstate for all threads in this zone. */
+        cpu_uarray_t    *zone_ustate;
         /* fork-fail kstat tracking */
         uint32_t        zone_ffcap;             /* hit an rctl cap */
         uint32_t        zone_ffnoproc;          /* get proc/lwp error */
         uint32_t        zone_ffnomem;           /* as_dup/memory error */
         uint32_t        zone_ffmisc;            /* misc. other error */