1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #include <sys/errno.h>
  28 #include <sys/cpuvar.h>
  29 #include <sys/stat.h>
  30 #include <sys/modctl.h>
  31 #include <sys/cmn_err.h>
  32 #include <sys/ddi.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/ksynch.h>
  35 #include <sys/conf.h>
  36 #include <sys/kmem.h>
  37 #include <sys/kcpc.h>
  38 #include <sys/cap_util.h>
  39 #include <sys/cpc_pcbe.h>
  40 #include <sys/cpc_impl.h>
  41 #include <sys/dtrace_impl.h>
  42 
  43 /*
  44  * DTrace CPU Performance Counter Provider
  45  * ---------------------------------------
  46  *
  47  * The DTrace cpc provider allows DTrace consumers to access the CPU
  48  * performance counter overflow mechanism of a CPU. The configuration
  49  * presented in a probe specification is programmed into the performance
  50  * counter hardware of all available CPUs on a system. Programming the
  51  * hardware causes a counter on each CPU to begin counting events of the
  52  * given type. When the specified number of events have occurred, an overflow
  53  * interrupt will be generated and the probe is fired.
  54  *
  55  * The required configuration for the performance counter is encoded into
  56  * the probe specification and this includes the performance counter event
  57  * name, processor mode, overflow rate and an optional unit mask.
  58  *
  59  * Most processors provide several counters (PICs) which can count all or a
  60  * subset of the events available for a given CPU. However, when overflow
  61  * profiling is being used, not all CPUs can detect which counter generated the
  62  * overflow interrupt. In this case we cannot reliably determine which counter
  63  * overflowed and we therefore only allow such CPUs to configure one event at
  64  * a time. Processors that can determine the counter which overflowed are
  65  * allowed to program as many events at one time as possible (in theory up to
  66  * the number of instrumentation counters supported by that platform).
  67  * Therefore, multiple consumers can enable multiple probes at the same time
  68  * on such platforms. Platforms which cannot determine the source of an
  69  * overflow interrupt are only allowed to program a single event at one time.
  70  *
  71  * The performance counter hardware is made available to consumers on a
  72  * first-come, first-served basis. Only a finite amount of hardware resource
  73  * is available and, while we make every attempt to accomodate requests from
  74  * consumers, we must deny requests when hardware resources have been exhausted.
  75  * A consumer will fail to enable probes when resources are currently in use.
  76  *
  77  * The cpc provider contends for shared hardware resources along with other
  78  * consumers of the kernel CPU performance counter subsystem (e.g. cpustat(1M)).
  79  * Only one such consumer can use the performance counters at any one time and
  80  * counters are made available on a first-come, first-served basis. As with
  81  * cpustat, the cpc provider has priority over per-LWP libcpc usage (e.g.
  82  * cputrack(1)). Invoking the cpc provider will cause all existing per-LWP
  83  * counter contexts to be invalidated.
  84  */
  85 
  86 typedef struct dcpc_probe {
  87         char            dcpc_event_name[CPC_MAX_EVENT_LEN];
  88         int             dcpc_flag;      /* flags (USER/SYS) */
  89         uint32_t        dcpc_ovfval;    /* overflow value */
  90         int64_t         dcpc_umask;     /* umask/emask for this event */
  91         int             dcpc_picno;     /* pic this event is programmed in */
  92         int             dcpc_enabled;   /* probe is actually enabled? */
  93         int             dcpc_disabling; /* probe is currently being disabled */
  94         dtrace_id_t     dcpc_id;        /* probeid this request is enabling */
  95         int             dcpc_actv_req_idx;      /* idx into dcpc_actv_reqs[] */
  96 } dcpc_probe_t;
  97 
  98 static dev_info_t                       *dcpc_devi;
  99 static dtrace_provider_id_t             dcpc_pid;
 100 static dcpc_probe_t                     **dcpc_actv_reqs;
 101 static uint32_t                         dcpc_enablings = 0;
 102 static int                              dcpc_ovf_mask = 0;
 103 static int                              dcpc_mult_ovf_cap = 0;
 104 static int                              dcpc_mask_type = 0;
 105 
 106 /*
 107  * When the dcpc provider is loaded, dcpc_min_overflow is set to either
 108  * DCPC_MIN_OVF_DEFAULT or the value that dcpc-min-overflow is set to in
 109  * the dcpc.conf file. Decrease this value to set probes with smaller
 110  * overflow values. Remember that very small values could render a system
 111  * unusable with frequently occurring events.
 112  */
 113 #define DCPC_MIN_OVF_DEFAULT            5000
 114 static uint32_t                         dcpc_min_overflow;
 115 
 116 static int dcpc_aframes = 0;    /* override for artificial frame setting */
 117 #if defined(__x86)
 118 #define DCPC_ARTIFICIAL_FRAMES  8
 119 #elif defined(__sparc)
 120 #define DCPC_ARTIFICIAL_FRAMES  2
 121 #endif
 122 
 123 /*
 124  * Called from the platform overflow interrupt handler. 'bitmap' is a mask
 125  * which contains the pic(s) that have overflowed.
 126  */
 127 static void
 128 dcpc_fire(uint64_t bitmap)
 129 {
 130         int i;
 131 
 132         /*
 133          * No counter was marked as overflowing. Shout about it and get out.
 134          */
 135         if ((bitmap & dcpc_ovf_mask) == 0) {
 136                 cmn_err(CE_NOTE, "dcpc_fire: no counter overflow found\n");
 137                 return;
 138         }
 139 
 140         /*
 141          * This is the common case of a processor that doesn't support
 142          * multiple overflow events. Such systems are only allowed a single
 143          * enabling and therefore we just look for the first entry in
 144          * the active request array.
 145          */
 146         if (!dcpc_mult_ovf_cap) {
 147                 for (i = 0; i < cpc_ncounters; i++) {
 148                         if (dcpc_actv_reqs[i] != NULL) {
 149                                 dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
 150                                     CPU->cpu_cpcprofile_pc,
 151                                     CPU->cpu_cpcprofile_upc, 0, 0, 0);
 152                                 return;
 153                         }
 154                 }
 155                 return;
 156         }
 157 
 158         /*
 159          * This is a processor capable of handling multiple overflow events.
 160          * Iterate over the array of active requests and locate the counters
 161          * that overflowed (note: it is possible for more than one counter to
 162          * have overflowed at the same time).
 163          */
 164         for (i = 0; i < cpc_ncounters; i++) {
 165                 if (dcpc_actv_reqs[i] != NULL &&
 166                     (bitmap & (1ULL << dcpc_actv_reqs[i]->dcpc_picno))) {
 167                         dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
 168                             CPU->cpu_cpcprofile_pc,
 169                             CPU->cpu_cpcprofile_upc, 0, 0, 0);
 170                 }
 171         }
 172 }
 173 
 174 static void
 175 dcpc_create_probe(dtrace_provider_id_t id, const char *probename,
 176     char *eventname, int64_t umask, uint32_t ovfval, char flag)
 177 {
 178         dcpc_probe_t *pp;
 179         int nr_frames = DCPC_ARTIFICIAL_FRAMES + dtrace_mach_aframes();
 180 
 181         if (dcpc_aframes)
 182                 nr_frames = dcpc_aframes;
 183 
 184         if (dtrace_probe_lookup(id, NULL, NULL, probename) != 0)
 185                 return;
 186 
 187         pp = kmem_zalloc(sizeof (dcpc_probe_t), KM_SLEEP);
 188         (void) strncpy(pp->dcpc_event_name, eventname,
 189             sizeof (pp->dcpc_event_name) - 1);
 190         pp->dcpc_event_name[sizeof (pp->dcpc_event_name) - 1] = '\0';
 191         pp->dcpc_flag = flag | CPC_OVF_NOTIFY_EMT;
 192         pp->dcpc_ovfval = ovfval;
 193         pp->dcpc_umask = umask;
 194         pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;
 195 
 196         pp->dcpc_id = dtrace_probe_create(id, NULL, NULL, probename,
 197             nr_frames, pp);
 198 }
 199 
 200 /*ARGSUSED*/
 201 static void
 202 dcpc_provide(void *arg, const dtrace_probedesc_t *desc)
 203 {
 204         /*
 205          * The format of a probe is:
 206          *
 207          *      event_name-mode-{optional_umask}-overflow_rate
 208          * e.g.
 209          *      DC_refill_from_system-user-0x1e-50000, or,
 210          *      DC_refill_from_system-all-10000
 211          *
 212          */
 213         char *str, *end, *p;
 214         int i, flag = 0;
 215         char event[CPC_MAX_EVENT_LEN];
 216         long umask = -1, val = 0;
 217         size_t evlen, len;
 218 
 219         /*
 220          * The 'cpc' provider offers no probes by default.
 221          */
 222         if (desc == NULL)
 223                 return;
 224 
 225         len = strlen(desc->dtpd_name);
 226         p = str = kmem_alloc(len + 1, KM_SLEEP);
 227         (void) strcpy(str, desc->dtpd_name);
 228 
 229         /*
 230          * We have a poor man's strtok() going on here. Replace any hyphens
 231          * in the the probe name with NULL characters in order to make it
 232          * easy to parse the string with regular string functions.
 233          */
 234         for (i = 0; i < len; i++) {
 235                 if (str[i] == '-')
 236                         str[i] = '\0';
 237         }
 238 
 239         /*
 240          * The first part of the string must be either a platform event
 241          * name or a generic event name.
 242          */
 243         evlen = strlen(p);
 244         (void) strncpy(event, p, CPC_MAX_EVENT_LEN - 1);
 245         event[CPC_MAX_EVENT_LEN - 1] = '\0';
 246 
 247         /*
 248          * The next part of the name is the mode specification. Valid
 249          * settings are "user", "kernel" or "all".
 250          */
 251         p += evlen + 1;
 252 
 253         if (strcmp(p, "user") == 0)
 254                 flag |= CPC_COUNT_USER;
 255         else if (strcmp(p, "kernel") == 0)
 256                 flag |= CPC_COUNT_SYSTEM;
 257         else if (strcmp(p, "all") == 0)
 258                 flag |= CPC_COUNT_USER | CPC_COUNT_SYSTEM;
 259         else
 260                 goto err;
 261 
 262         /*
 263          * Next we either have a mask specification followed by an overflow
 264          * rate or just an overflow rate on its own.
 265          */
 266         p += strlen(p) + 1;
 267         if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) {
 268                 /*
 269                  * A unit mask can only be specified if:
 270                  * 1) this performance counter back end supports masks.
 271                  * 2) the specified event is platform specific.
 272                  * 3) a valid hex number is converted.
 273                  * 4) no extraneous characters follow the mask specification.
 274                  */
 275                 if (dcpc_mask_type != 0 && strncmp(event, "PAPI", 4) != 0 &&
 276                     ddi_strtol(p, &end, 16, &umask) == 0 &&
 277                     end == p + strlen(p)) {
 278                         p += strlen(p) + 1;
 279                 } else {
 280                         goto err;
 281                 }
 282         }
 283 
 284         /*
 285          * This final part must be an overflow value which has to be greater
 286          * than the minimum permissible overflow rate.
 287          */
 288         if ((ddi_strtol(p, &end, 10, &val) != 0) || end != p + strlen(p) ||
 289             val < dcpc_min_overflow)
 290                 goto err;
 291 
 292         /*
 293          * Validate the event and create the probe.
 294          */
 295         for (i = 0; i < cpc_ncounters; i++) {
 296                 char *events, *cp, *p, *end;
 297                 int found = 0, j;
 298                 size_t llen;
 299 
 300                 if ((events = kcpc_list_events(i)) == NULL)
 301                         goto err;
 302 
 303                 llen = strlen(events);
 304                 p = cp = ddi_strdup(events, KM_NOSLEEP);
 305                 end = cp + llen;
 306 
 307                 for (j = 0; j < llen; j++) {
 308                         if (cp[j] == ',')
 309                                 cp[j] = '\0';
 310                 }
 311 
 312                 while (p < end && found == 0) {
 313                         if (strcmp(p, event) == 0) {
 314                                 dcpc_create_probe(dcpc_pid, desc->dtpd_name,
 315                                     event, umask, (uint32_t)val, flag);
 316                                 found = 1;
 317                         }
 318                         p += strlen(p) + 1;
 319                 }
 320                 kmem_free(cp, llen + 1);
 321 
 322                 if (found)
 323                         break;
 324         }
 325 
 326 err:
 327         kmem_free(str, len + 1);
 328 }
 329 
 330 /*ARGSUSED*/
 331 static void
 332 dcpc_destroy(void *arg, dtrace_id_t id, void *parg)
 333 {
 334         dcpc_probe_t *pp = parg;
 335 
 336         ASSERT(pp->dcpc_enabled == 0);
 337         kmem_free(pp, sizeof (dcpc_probe_t));
 338 }
 339 
 340 /*ARGSUSED*/
 341 static int
 342 dcpc_mode(void *arg, dtrace_id_t id, void *parg)
 343 {
 344         if (CPU->cpu_cpcprofile_pc == 0) {
 345                 return (DTRACE_MODE_NOPRIV_DROP | DTRACE_MODE_USER);
 346         } else {
 347                 return (DTRACE_MODE_NOPRIV_DROP | DTRACE_MODE_KERNEL);
 348         }
 349 }
 350 
 351 static void
 352 dcpc_populate_set(cpu_t *c, dcpc_probe_t *pp, kcpc_set_t *set, int reqno)
 353 {
 354         kcpc_set_t *oset;
 355         int i;
 356 
 357         (void) strncpy(set->ks_req[reqno].kr_event, pp->dcpc_event_name,
 358             CPC_MAX_EVENT_LEN);
 359         set->ks_req[reqno].kr_config = NULL;
 360         set->ks_req[reqno].kr_index = reqno;
 361         set->ks_req[reqno].kr_picnum = -1;
 362         set->ks_req[reqno].kr_flags =  pp->dcpc_flag;
 363 
 364         /*
 365          * If a unit mask has been specified then detect which attribute
 366          * the platform needs. For now, it's either "umask" or "emask".
 367          */
 368         if (pp->dcpc_umask >= 0) {
 369                 set->ks_req[reqno].kr_attr =
 370                     kmem_zalloc(sizeof (kcpc_attr_t), KM_SLEEP);
 371                 set->ks_req[reqno].kr_nattrs = 1;
 372                 if (dcpc_mask_type & DCPC_UMASK)
 373                         (void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
 374                             "umask", 5);
 375                 else
 376                         (void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
 377                             "emask", 5);
 378                 set->ks_req[reqno].kr_attr->ka_val = pp->dcpc_umask;
 379         } else {
 380                 set->ks_req[reqno].kr_attr = NULL;
 381                 set->ks_req[reqno].kr_nattrs = 0;
 382         }
 383 
 384         /*
 385          * If this probe is enabled, obtain its current countdown value
 386          * and use that. The CPUs cpc context might not exist yet if we
 387          * are dealing with a CPU that is just coming online.
 388          */
 389         if (pp->dcpc_enabled && (c->cpu_cpc_ctx != NULL)) {
 390                 oset = c->cpu_cpc_ctx->kc_set;
 391 
 392                 for (i = 0; i < oset->ks_nreqs; i++) {
 393                         if (strcmp(oset->ks_req[i].kr_event,
 394                             set->ks_req[reqno].kr_event) == 0) {
 395                                 set->ks_req[reqno].kr_preset =
 396                                     *(oset->ks_req[i].kr_data);
 397                         }
 398                 }
 399         } else {
 400                 set->ks_req[reqno].kr_preset = UINT64_MAX - pp->dcpc_ovfval;
 401         }
 402 
 403         set->ks_nreqs++;
 404 }
 405 
 406 
 407 /*
 408  * Create a fresh request set for the enablings represented in the
 409  * 'dcpc_actv_reqs' array which contains the probes we want to be
 410  * in the set. This can be called for several reasons:
 411  *
 412  * 1)   We are on a single or multi overflow platform and we have no
 413  *      current events so we can just create the set and initialize it.
 414  * 2)   We are on a multi-overflow platform and we already have one or
 415  *      more existing events and we are adding a new enabling. Create a
 416  *      new set and copy old requests in and then add the new request.
 417  * 3)   We are on a multi-overflow platform and we have just removed an
 418  *      enabling but we still have enablings whch are valid. Create a new
 419  *      set and copy in still valid requests.
 420  */
 421 static kcpc_set_t *
 422 dcpc_create_set(cpu_t *c)
 423 {
 424         int i, reqno = 0;
 425         int active_requests = 0;
 426         kcpc_set_t *set;
 427 
 428         /*
 429          * First get a count of the number of currently active requests.
 430          * Note that dcpc_actv_reqs[] should always reflect which requests
 431          * we want to be in the set that is to be created. It is the
 432          * responsibility of the caller of dcpc_create_set() to adjust that
 433          * array accordingly beforehand.
 434          */
 435         for (i = 0; i < cpc_ncounters; i++) {
 436                 if (dcpc_actv_reqs[i] != NULL)
 437                         active_requests++;
 438         }
 439 
 440         set = kmem_zalloc(sizeof (kcpc_set_t), KM_SLEEP);
 441 
 442         set->ks_req =
 443             kmem_zalloc(sizeof (kcpc_request_t) * active_requests, KM_SLEEP);
 444 
 445         set->ks_data =
 446             kmem_zalloc(active_requests * sizeof (uint64_t), KM_SLEEP);
 447 
 448         /*
 449          * Look for valid entries in the active requests array and populate
 450          * the request set for any entries found.
 451          */
 452         for (i = 0; i < cpc_ncounters; i++) {
 453                 if (dcpc_actv_reqs[i] != NULL) {
 454                         dcpc_populate_set(c, dcpc_actv_reqs[i], set, reqno);
 455                         reqno++;
 456                 }
 457         }
 458 
 459         return (set);
 460 }
 461 
 462 static int
 463 dcpc_program_cpu_event(cpu_t *c)
 464 {
 465         int i, j, subcode;
 466         kcpc_ctx_t *ctx, *octx;
 467         kcpc_set_t *set;
 468 
 469         set = dcpc_create_set(c);
 470 
 471         set->ks_ctx = ctx = kcpc_ctx_alloc(KM_SLEEP);
 472         ctx->kc_set = set;
 473         ctx->kc_cpuid = c->cpu_id;
 474 
 475         if (kcpc_assign_reqs(set, ctx) != 0)
 476                 goto err;
 477 
 478         if (kcpc_configure_reqs(ctx, set, &subcode) != 0)
 479                 goto err;
 480 
 481         for (i = 0; i < set->ks_nreqs; i++) {
 482                 for (j = 0; j < cpc_ncounters; j++) {
 483                         if (dcpc_actv_reqs[j] != NULL &&
 484                             strcmp(set->ks_req[i].kr_event,
 485                             dcpc_actv_reqs[j]->dcpc_event_name) == 0) {
 486                                 dcpc_actv_reqs[j]->dcpc_picno =
 487                                     set->ks_req[i].kr_picnum;
 488                         }
 489                 }
 490         }
 491 
 492         /*
 493          * If we already have an active enabling then save the current cpc
 494          * context away.
 495          */
 496         octx = c->cpu_cpc_ctx;
 497 
 498         kcpc_cpu_program(c, ctx);
 499 
 500         if (octx != NULL) {
 501                 kcpc_set_t *oset = octx->kc_set;
 502                 kmem_free(oset->ks_data, oset->ks_nreqs * sizeof (uint64_t));
 503                 kcpc_free_configs(oset);
 504                 kcpc_free_set(oset);
 505                 kcpc_ctx_free(octx);
 506         }
 507 
 508         return (0);
 509 
 510 err:
 511         /*
 512          * We failed to configure this request up so free things up and
 513          * get out.
 514          */
 515         kcpc_free_configs(set);
 516         kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
 517         kcpc_free_set(set);
 518         kcpc_ctx_free(ctx);
 519 
 520         return (-1);
 521 }
 522 
 523 static void
 524 dcpc_disable_cpu(cpu_t *c)
 525 {
 526         kcpc_ctx_t *ctx;
 527         kcpc_set_t *set;
 528 
 529         /*
 530          * Leave this CPU alone if it's already offline.
 531          */
 532         if (c->cpu_flags & CPU_OFFLINE)
 533                 return;
 534 
 535         /*
 536          * Grab CPUs CPC context before kcpc_cpu_stop() stops counters and
 537          * changes it.
 538          */
 539         ctx = c->cpu_cpc_ctx;
 540 
 541         kcpc_cpu_stop(c, B_FALSE);
 542 
 543         set = ctx->kc_set;
 544 
 545         kcpc_free_configs(set);
 546         kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
 547         kcpc_free_set(set);
 548         kcpc_ctx_free(ctx);
 549 }
 550 
 551 /*
 552  * The dcpc_*_interrupts() routines are responsible for manipulating the
 553  * per-CPU dcpc interrupt state byte. The purpose of the state byte is to
 554  * synchronize processing of hardware overflow interrupts wth configuration
 555  * changes made to the CPU performance counter subsystem by the dcpc provider.
 556  *
 557  * The dcpc provider claims ownership of the overflow interrupt mechanism
 558  * by transitioning the state byte from DCPC_INTR_INACTIVE (indicating the
 559  * dcpc provider is not in use) to DCPC_INTR_FREE (the dcpc provider owns the
 560  * overflow mechanism and interrupts may be processed). Before modifying
 561  * a CPUs configuration state the state byte is transitioned from
 562  * DCPC_INTR_FREE to DCPC_INTR_CONFIG ("configuration in process" state).
 563  * The hardware overflow handler, kcpc_hw_overflow_intr(), will only process
 564  * an interrupt when a configuration is not in process (i.e. the state is
 565  * marked as free). During interrupt processing the state is set to
 566  * DCPC_INTR_PROCESSING by the overflow handler. When the last dcpc based
 567  * enabling is removed, the state byte is set to DCPC_INTR_INACTIVE to indicate
 568  * the dcpc provider is no longer interested in overflow interrupts.
 569  */
 570 static void
 571 dcpc_block_interrupts(void)
 572 {
 573         cpu_t *c = cpu_list;
 574         uint8_t *state;
 575 
 576         ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);
 577 
 578         do {
 579                 state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;
 580 
 581                 while (atomic_cas_8(state, DCPC_INTR_FREE,
 582                     DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
 583                         continue;
 584 
 585         } while ((c = c->cpu_next) != cpu_list);
 586 }
 587 
 588 /*
 589  * Set all CPUs dcpc interrupt state to DCPC_INTR_FREE to indicate that
 590  * overflow interrupts can be processed safely.
 591  */
 592 static void
 593 dcpc_release_interrupts(void)
 594 {
 595         cpu_t *c = cpu_list;
 596 
 597         ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);
 598 
 599         do {
 600                 cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_FREE;
 601                 membar_producer();
 602         } while ((c = c->cpu_next) != cpu_list);
 603 }
 604 
 605 /*
 606  * Transition all CPUs dcpc interrupt state from DCPC_INTR_INACTIVE to
 607  * to DCPC_INTR_FREE. This indicates that the dcpc provider is now
 608  * responsible for handling all overflow interrupt activity. Should only be
 609  * called before enabling the first dcpc based probe.
 610  */
 611 static void
 612 dcpc_claim_interrupts(void)
 613 {
 614         cpu_t *c = cpu_list;
 615 
 616         ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state == DCPC_INTR_INACTIVE);
 617 
 618         do {
 619                 cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_FREE;
 620                 membar_producer();
 621         } while ((c = c->cpu_next) != cpu_list);
 622 }
 623 
 624 /*
 625  * Set all CPUs dcpc interrupt state to DCPC_INTR_INACTIVE to indicate that
 626  * the dcpc provider is no longer processing overflow interrupts. Only called
 627  * during removal of the last dcpc based enabling.
 628  */
 629 static void
 630 dcpc_surrender_interrupts(void)
 631 {
 632         cpu_t *c = cpu_list;
 633 
 634         ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);
 635 
 636         do {
 637                 cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_INACTIVE;
 638                 membar_producer();
 639         } while ((c = c->cpu_next) != cpu_list);
 640 }
 641 
 642 /*
 643  * dcpc_program_event() can be called owing to a new enabling or if a multi
 644  * overflow platform has disabled a request but needs to  program the requests
 645  * that are still valid.
 646  *
 647  * Every invocation of dcpc_program_event() will create a new kcpc_ctx_t
 648  * and a new request set which contains the new enabling and any old enablings
 649  * which are still valid (possible with multi-overflow platforms).
 650  */
 651 static int
 652 dcpc_program_event(dcpc_probe_t *pp)
 653 {
 654         cpu_t *c;
 655         int ret = 0;
 656 
 657         ASSERT(MUTEX_HELD(&cpu_lock));
 658 
 659         kpreempt_disable();
 660 
 661         dcpc_block_interrupts();
 662 
 663         c = cpu_list;
 664 
 665         do {
 666                 /*
 667                  * Skip CPUs that are currently offline.
 668                  */
 669                 if (c->cpu_flags & CPU_OFFLINE)
 670                         continue;
 671 
 672                 /*
 673                  * Stop counters but preserve existing DTrace CPC context
 674                  * if there is one.
 675                  *
 676                  * If we come here when the first event is programmed for a CPU,
 677                  * there should be no DTrace CPC context installed. In this
 678                  * case, kcpc_cpu_stop() will ensure that there is no other
 679                  * context on the CPU.
 680                  *
 681                  * If we add new enabling to the original one, the CPU should
 682                  * have the old DTrace CPC context which we need to keep around
 683                  * since dcpc_program_event() will add to it.
 684                  */
 685                 if (c->cpu_cpc_ctx != NULL)
 686                         kcpc_cpu_stop(c, B_TRUE);
 687         } while ((c = c->cpu_next) != cpu_list);
 688 
 689         dcpc_release_interrupts();
 690 
 691         /*
 692          * If this enabling is being removed (in the case of a multi event
 693          * capable system with more than one active enabling), we can now
 694          * update the active request array to reflect the enablings that need
 695          * to be reprogrammed.
 696          */
 697         if (pp->dcpc_disabling == 1)
 698                 dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
 699 
 700         do {
 701                 /*
 702                  * Skip CPUs that are currently offline.
 703                  */
 704                 if (c->cpu_flags & CPU_OFFLINE)
 705                         continue;
 706 
 707                 ret = dcpc_program_cpu_event(c);
 708         } while ((c = c->cpu_next) != cpu_list && ret == 0);
 709 
 710         /*
 711          * If dcpc_program_cpu_event() fails then it is because we couldn't
 712          * configure the requests in the set for the CPU and not because of
 713          * an error programming the hardware. If we have a failure here then
 714          * we assume no CPUs have been programmed in the above step as they
 715          * are all configured identically.
 716          */
 717         if (ret != 0) {
 718                 pp->dcpc_enabled = 0;
 719                 kpreempt_enable();
 720                 return (-1);
 721         }
 722 
 723         if (pp->dcpc_disabling != 1)
 724                 pp->dcpc_enabled = 1;
 725 
 726         kpreempt_enable();
 727 
 728         return (0);
 729 }
 730 
 731 /*ARGSUSED*/
 732 static int
 733 dcpc_enable(void *arg, dtrace_id_t id, void *parg)
 734 {
 735         dcpc_probe_t *pp = parg;
 736         int i, found = 0;
 737         cpu_t *c;
 738 
 739         ASSERT(MUTEX_HELD(&cpu_lock));
 740 
 741         /*
 742          * Bail out if the counters are being used by a libcpc consumer.
 743          */
 744         rw_enter(&kcpc_cpuctx_lock, RW_READER);
 745         if (kcpc_cpuctx > 0) {
 746                 rw_exit(&kcpc_cpuctx_lock);
 747                 return (-1);
 748         }
 749 
 750         dtrace_cpc_in_use++;
 751         rw_exit(&kcpc_cpuctx_lock);
 752 
 753         /*
 754          * Locate this enabling in the first free entry of the active
 755          * request array.
 756          */
 757         for (i = 0; i < cpc_ncounters; i++) {
 758                 if (dcpc_actv_reqs[i] == NULL) {
 759                         dcpc_actv_reqs[i] = pp;
 760                         pp->dcpc_actv_req_idx = i;
 761                         found = 1;
 762                         break;
 763                 }
 764         }
 765 
 766         /*
 767          * If we couldn't find a slot for this probe then there is no
 768          * room at the inn.
 769          */
 770         if (!found) {
 771                 dtrace_cpc_in_use--;
 772                 return (-1);
 773         }
 774 
 775         ASSERT(pp->dcpc_actv_req_idx >= 0);
 776 
 777         /*
 778          * DTrace is taking over CPC contexts, so stop collecting
 779          * capacity/utilization data for all CPUs.
 780          */
 781         if (dtrace_cpc_in_use == 1)
 782                 cu_disable();
 783 
 784         /*
 785          * The following must hold true if we are to (attempt to) enable
 786          * this request:
 787          *
 788          * 1) No enablings currently exist. We allow all platforms to
 789          * proceed if this is true.
 790          *
 791          * OR
 792          *
 793          * 2) If the platform is multi overflow capable and there are
 794          * less valid enablings than there are counters. There is no
 795          * guarantee that a platform can accommodate as many events as
 796          * it has counters for but we will at least try to program
 797          * up to that many requests.
 798          *
 799          * The 'dcpc_enablings' variable is implictly protected by locking
 800          * provided by the DTrace framework and the cpu management framework.
 801          */
 802         if (dcpc_enablings == 0 || (dcpc_mult_ovf_cap &&
 803             dcpc_enablings < cpc_ncounters)) {
 804                 /*
 805                  * Before attempting to program the first enabling we need to
 806                  * invalidate any lwp-based contexts and lay claim to the
 807                  * overflow interrupt mechanism.
 808                  */
 809                 if (dcpc_enablings == 0) {
 810                         kcpc_invalidate_all();
 811                         dcpc_claim_interrupts();
 812                 }
 813 
 814                 if (dcpc_program_event(pp) == 0) {
 815                         dcpc_enablings++;
 816                         return (0);
 817                 }
 818         }
 819 
 820         /*
 821          * If active enablings existed before we failed to enable this probe
 822          * on a multi event capable platform then we need to restart counters
 823          * as they will have been stopped in the attempted configuration. The
 824          * context should now just contain the request prior to this failed
 825          * enabling.
 826          */
 827         if (dcpc_enablings > 0 && dcpc_mult_ovf_cap) {
 828                 c = cpu_list;
 829 
 830                 ASSERT(dcpc_mult_ovf_cap == 1);
 831                 do {
 832                         /*
 833                          * Skip CPUs that are currently offline.
 834                          */
 835                         if (c->cpu_flags & CPU_OFFLINE)
 836                                 continue;
 837 
 838                         kcpc_cpu_program(c, c->cpu_cpc_ctx);
 839                 } while ((c = c->cpu_next) != cpu_list);
 840         }
 841 
 842         /*
 843          * Give up any claim to the overflow interrupt mechanism if no
 844          * dcpc based enablings exist.
 845          */
 846         if (dcpc_enablings == 0)
 847                 dcpc_surrender_interrupts();
 848 
 849         dtrace_cpc_in_use--;
 850         dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
 851         pp->dcpc_actv_req_idx = pp->dcpc_picno = -1;
 852 
 853         /*
 854          * If all probes are removed, enable capacity/utilization data
 855          * collection for every CPU.
 856          */
 857         if (dtrace_cpc_in_use == 0)
 858                 cu_enable();
 859 
 860         return (-1);
 861 }
 862 
 863 /*
 864  * If only one enabling is active then remove the context and free
 865  * everything up. If there are multiple enablings active then remove this
 866  * one, its associated meta-data and re-program the hardware.
 867  */
 868 /*ARGSUSED*/
 869 static void
 870 dcpc_disable(void *arg, dtrace_id_t id, void *parg)
 871 {
 872         cpu_t *c;
 873         dcpc_probe_t *pp = parg;
 874 
 875         ASSERT(MUTEX_HELD(&cpu_lock));
 876 
 877         kpreempt_disable();
 878 
 879         /*
 880          * This probe didn't actually make it as far as being fully enabled
 881          * so we needn't do anything with it.
 882          */
 883         if (pp->dcpc_enabled == 0) {
 884                 /*
 885                  * If we actually allocated this request a slot in the
 886                  * request array but failed to enabled it then remove the
 887                  * entry in the array.
 888                  */
 889                 if (pp->dcpc_actv_req_idx >= 0) {
 890                         dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
 891                         pp->dcpc_actv_req_idx = pp->dcpc_picno =
 892                             pp->dcpc_disabling = -1;
 893                 }
 894 
 895                 kpreempt_enable();
 896                 return;
 897         }
 898 
 899         /*
 900          * If this is the only enabling then stop all the counters and
 901          * free up the meta-data.
 902          */
 903         if (dcpc_enablings == 1) {
 904                 ASSERT(dtrace_cpc_in_use == 1);
 905 
 906                 dcpc_block_interrupts();
 907 
 908                 c = cpu_list;
 909 
 910                 do {
 911                         dcpc_disable_cpu(c);
 912                 } while ((c = c->cpu_next) != cpu_list);
 913 
 914                 dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
 915                 dcpc_surrender_interrupts();
 916         } else {
 917                 /*
 918                  * This platform can support multiple overflow events and
 919                  * the enabling being disabled is not the last one. Remove this
 920                  * enabling and re-program the hardware with the new config.
 921                  */
 922                 ASSERT(dcpc_mult_ovf_cap);
 923                 ASSERT(dcpc_enablings > 1);
 924 
 925                 pp->dcpc_disabling = 1;
 926                 (void) dcpc_program_event(pp);
 927         }
 928 
 929         kpreempt_enable();
 930 
 931         dcpc_enablings--;
 932         dtrace_cpc_in_use--;
 933         pp->dcpc_enabled = 0;
 934         pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;
 935 
 936         /*
 937          * If all probes are removed, enable capacity/utilization data
 938          * collection for every CPU
 939          */
 940         if (dtrace_cpc_in_use == 0)
 941                 cu_enable();
 942 }
 943 
 944 /*ARGSUSED*/
 945 static int
 946 dcpc_cpu_setup(cpu_setup_t what, processorid_t cpu, void *arg)
 947 {
 948         cpu_t *c;
 949         uint8_t *state;
 950 
 951         ASSERT(MUTEX_HELD(&cpu_lock));
 952 
 953         switch (what) {
 954         case CPU_OFF:
 955                 /*
 956                  * Offline CPUs are not allowed to take part so remove this
 957                  * CPU if we are actively tracing.
 958                  */
 959                 if (dtrace_cpc_in_use) {
 960                         c = cpu_get(cpu);
 961                         state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;
 962 
 963                         /*
 964                          * Indicate that a configuration is in process in
 965                          * order to stop overflow interrupts being processed
 966                          * on this CPU while we disable it.
 967                          */
 968                         while (atomic_cas_8(state, DCPC_INTR_FREE,
 969                             DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
 970                                 continue;
 971 
 972                         dcpc_disable_cpu(c);
 973 
 974                         /*
 975                          * Reset this CPUs interrupt state as the configuration
 976                          * has ended.
 977                          */
 978                         cpu_core[c->cpu_id].cpuc_dcpc_intr_state =
 979                             DCPC_INTR_FREE;
 980                         membar_producer();
 981                 }
 982                 break;
 983 
 984         case CPU_ON:
 985         case CPU_SETUP:
 986                 /*
 987                  * This CPU is being initialized or brought online so program
 988                  * it with the current request set if we are actively tracing.
 989                  */
 990                 if (dtrace_cpc_in_use) {
 991                         c = cpu_get(cpu);
 992                         (void) dcpc_program_cpu_event(c);
 993                 }
 994                 break;
 995 
 996         default:
 997                 break;
 998         }
 999 
1000         return (0);
1001 }
1002 
1003 static dtrace_pattr_t dcpc_attr = {
1004 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
1005 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1006 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1007 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_CPU },
1008 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
1009 };
1010 
1011 static dtrace_pops_t dcpc_pops = {
1012     dcpc_provide,
1013     NULL,
1014     dcpc_enable,
1015     dcpc_disable,
1016     NULL,
1017     NULL,
1018     NULL,
1019     NULL,
1020     dcpc_mode,
1021     dcpc_destroy
1022 };
1023 
1024 /*ARGSUSED*/
1025 static int
1026 dcpc_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1027 {
1028         return (0);
1029 }
1030 
1031 /*ARGSUSED*/
1032 static int
1033 dcpc_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1034 {
1035         int error;
1036 
1037         switch (infocmd) {
1038         case DDI_INFO_DEVT2DEVINFO:
1039                 *result = (void *)dcpc_devi;
1040                 error = DDI_SUCCESS;
1041                 break;
1042         case DDI_INFO_DEVT2INSTANCE:
1043                 *result = (void *)0;
1044                 error = DDI_SUCCESS;
1045                 break;
1046         default:
1047                 error = DDI_FAILURE;
1048         }
1049         return (error);
1050 }
1051 
1052 static int
1053 dcpc_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
1054 {
1055         switch (cmd) {
1056         case DDI_DETACH:
1057                 break;
1058         case DDI_SUSPEND:
1059                 return (DDI_SUCCESS);
1060         default:
1061                 return (DDI_FAILURE);
1062         }
1063 
1064         if (dtrace_unregister(dcpc_pid) != 0)
1065                 return (DDI_FAILURE);
1066 
1067         ddi_remove_minor_node(devi, NULL);
1068 
1069         mutex_enter(&cpu_lock);
1070         unregister_cpu_setup_func(dcpc_cpu_setup, NULL);
1071         mutex_exit(&cpu_lock);
1072 
1073         kmem_free(dcpc_actv_reqs, cpc_ncounters * sizeof (dcpc_probe_t *));
1074 
1075         kcpc_unregister_dcpc();
1076 
1077         return (DDI_SUCCESS);
1078 }
1079 
1080 static int
1081 dcpc_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1082 {
1083         uint_t caps;
1084         char *attrs;
1085 
1086         switch (cmd) {
1087         case DDI_ATTACH:
1088                 break;
1089         case DDI_RESUME:
1090                 return (DDI_SUCCESS);
1091         default:
1092                 return (DDI_FAILURE);
1093         }
1094 
1095         if (kcpc_pcbe_loaded() == -1)
1096                 return (DDI_FAILURE);
1097 
1098         caps = kcpc_pcbe_capabilities();
1099 
1100         if (!(caps & CPC_CAP_OVERFLOW_INTERRUPT)) {
1101                 cmn_err(CE_NOTE, "!dcpc: Counter Overflow not supported"\
1102                     " on this processor");
1103                 return (DDI_FAILURE);
1104         }
1105 
1106         if (ddi_create_minor_node(devi, "dcpc", S_IFCHR, 0,
1107             DDI_PSEUDO, NULL) == DDI_FAILURE ||
1108             dtrace_register("cpc", &dcpc_attr, DTRACE_PRIV_KERNEL,
1109             NULL, &dcpc_pops, NULL, &dcpc_pid) != 0) {
1110                 ddi_remove_minor_node(devi, NULL);
1111                 return (DDI_FAILURE);
1112         }
1113 
1114         mutex_enter(&cpu_lock);
1115         register_cpu_setup_func(dcpc_cpu_setup, NULL);
1116         mutex_exit(&cpu_lock);
1117 
1118         dcpc_ovf_mask = (1 << cpc_ncounters) - 1;
1119         ASSERT(dcpc_ovf_mask != 0);
1120 
1121         if (caps & CPC_CAP_OVERFLOW_PRECISE)
1122                 dcpc_mult_ovf_cap = 1;
1123 
1124         /*
1125          * Determine which, if any, mask attribute the back-end can use.
1126          */
1127         attrs = kcpc_list_attrs();
1128         if (strstr(attrs, "umask") != NULL)
1129                 dcpc_mask_type |= DCPC_UMASK;
1130         else if (strstr(attrs, "emask") != NULL)
1131                 dcpc_mask_type |= DCPC_EMASK;
1132 
1133         /*
1134          * The dcpc_actv_reqs array is used to store the requests that
1135          * we currently have programmed. The order of requests in this
1136          * array is not necessarily the order that the event appears in
1137          * the kcpc_request_t array. Once entered into a slot in the array
1138          * the entry is not moved until it's removed.
1139          */
1140         dcpc_actv_reqs =
1141             kmem_zalloc(cpc_ncounters * sizeof (dcpc_probe_t *), KM_SLEEP);
1142 
1143         dcpc_min_overflow = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
1144             DDI_PROP_DONTPASS, "dcpc-min-overflow", DCPC_MIN_OVF_DEFAULT);
1145 
1146         kcpc_register_dcpc(dcpc_fire);
1147 
1148         ddi_report_dev(devi);
1149         dcpc_devi = devi;
1150 
1151         return (DDI_SUCCESS);
1152 }
1153 
1154 static struct cb_ops dcpc_cb_ops = {
1155         dcpc_open,              /* open */
1156         nodev,                  /* close */
1157         nulldev,                /* strategy */
1158         nulldev,                /* print */
1159         nodev,                  /* dump */
1160         nodev,                  /* read */
1161         nodev,                  /* write */
1162         nodev,                  /* ioctl */
1163         nodev,                  /* devmap */
1164         nodev,                  /* mmap */
1165         nodev,                  /* segmap */
1166         nochpoll,               /* poll */
1167         ddi_prop_op,            /* cb_prop_op */
1168         0,                      /* streamtab  */
1169         D_NEW | D_MP            /* Driver compatibility flag */
1170 };
1171 
1172 static struct dev_ops dcpc_ops = {
1173         DEVO_REV,               /* devo_rev, */
1174         0,                      /* refcnt  */
1175         dcpc_info,              /* get_dev_info */
1176         nulldev,                /* identify */
1177         nulldev,                /* probe */
1178         dcpc_attach,            /* attach */
1179         dcpc_detach,            /* detach */
1180         nodev,                  /* reset */
1181         &dcpc_cb_ops,               /* driver operations */
1182         NULL,                   /* bus operations */
1183         nodev,                  /* dev power */
1184         ddi_quiesce_not_needed  /* quiesce */
1185 };
1186 
1187 /*
1188  * Module linkage information for the kernel.
1189  */
1190 static struct modldrv modldrv = {
1191         &mod_driverops,             /* module type */
1192         "DTrace CPC Module",    /* name of module */
1193         &dcpc_ops,          /* driver ops */
1194 };
1195 
1196 static struct modlinkage modlinkage = {
1197         MODREV_1,
1198         (void *)&modldrv,
1199         NULL
1200 };
1201 
1202 int
1203 _init(void)
1204 {
1205         return (mod_install(&modlinkage));
1206 }
1207 
1208 int
1209 _info(struct modinfo *modinfop)
1210 {
1211         return (mod_info(&modlinkage, modinfop));
1212 }
1213 
1214 int
1215 _fini(void)
1216 {
1217         return (mod_remove(&modlinkage));
1218 }