5702 break;
5703 /*FALLTHROUGH*/
5704 case DIF_OP_LDUH:
5705 regs[rd] = dtrace_load16(regs[r1]);
5706 break;
5707 case DIF_OP_RLDUW:
5708 if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5709 break;
5710 /*FALLTHROUGH*/
5711 case DIF_OP_LDUW:
5712 regs[rd] = dtrace_load32(regs[r1]);
5713 break;
5714 case DIF_OP_RLDX:
5715 if (!dtrace_canload(regs[r1], 8, mstate, vstate))
5716 break;
5717 /*FALLTHROUGH*/
5718 case DIF_OP_LDX:
5719 regs[rd] = dtrace_load64(regs[r1]);
5720 break;
5721 case DIF_OP_ULDSB:
5722 regs[rd] = (int8_t)
5723 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5724 break;
5725 case DIF_OP_ULDSH:
5726 regs[rd] = (int16_t)
5727 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5728 break;
5729 case DIF_OP_ULDSW:
5730 regs[rd] = (int32_t)
5731 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5732 break;
5733 case DIF_OP_ULDUB:
5734 regs[rd] =
5735 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5736 break;
5737 case DIF_OP_ULDUH:
5738 regs[rd] =
5739 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5740 break;
5741 case DIF_OP_ULDUW:
5742 regs[rd] =
5743 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5744 break;
5745 case DIF_OP_ULDX:
5746 regs[rd] =
5747 dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5748 break;
5749 case DIF_OP_RET:
5750 rval = regs[rd];
5751 pc = textlen;
5752 break;
5753 case DIF_OP_NOP:
5754 break;
5755 case DIF_OP_SETX:
5756 regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5757 break;
5758 case DIF_OP_SETS:
5759 regs[rd] = (uint64_t)(uintptr_t)
5760 (strtab + DIF_INSTR_STRING(instr));
5761 break;
5762 case DIF_OP_SCMP: {
5763 size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5764 uintptr_t s1 = regs[r1];
5765 uintptr_t s2 = regs[r2];
5766
5767 if (s1 != NULL &&
6535 }
6536
6537 if (offs >= strsize) {
6538 /*
6539 * If we didn't have room for all of the strings, we don't
6540 * abort processing -- this needn't be a fatal error -- but we
6541 * still want to increment a counter (dts_stkstroverflows) to
6542 * allow this condition to be warned about. (If this is from
6543 * a jstack() action, it is easily tuned via jstackstrsize.)
6544 */
6545 dtrace_error(&state->dts_stkstroverflows);
6546 }
6547
6548 while (offs < strsize)
6549 str[offs++] = '\0';
6550
6551 out:
6552 mstate->dtms_scratch_ptr = old;
6553 }
6554
6555 /*
6556 * If you're looking for the epicenter of DTrace, you just found it. This
6557 * is the function called by the provider to fire a probe -- from which all
6558 * subsequent probe-context DTrace activity emanates.
6559 */
6560 void
6561 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
6562 uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
6563 {
6564 processorid_t cpuid;
6565 dtrace_icookie_t cookie;
6566 dtrace_probe_t *probe;
6567 dtrace_mstate_t mstate;
6568 dtrace_ecb_t *ecb;
6569 dtrace_action_t *act;
6570 intptr_t offs;
6571 size_t size;
6572 int vtime, onintr;
6573 volatile uint16_t *flags;
6574 hrtime_t now, end;
7026
7027 if (current == DTRACE_ACTIVITY_COOLDOWN)
7028 break;
7029
7030 if (current != DTRACE_ACTIVITY_WARMUP)
7031 current = DTRACE_ACTIVITY_ACTIVE;
7032
7033 if (dtrace_cas32(activity, current,
7034 DTRACE_ACTIVITY_DRAINING) != current) {
7035 *flags |= CPU_DTRACE_DROP;
7036 continue;
7037 }
7038
7039 break;
7040 }
7041
7042 default:
7043 ASSERT(0);
7044 }
7045
7046 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
7047 uintptr_t end = valoffs + size;
7048
7049 if (tracememsize != 0 &&
7050 valoffs + tracememsize < end) {
7051 end = valoffs + tracememsize;
7052 tracememsize = 0;
7053 }
7054
7055 if (!dtrace_vcanload((void *)(uintptr_t)val,
7056 &dp->dtdo_rtype, &mstate, vstate))
7057 continue;
7058
7059 /*
7060 * If this is a string, we're going to only
7061 * load until we find the zero byte -- after
7062 * which we'll store zero bytes.
7063 */
7064 if (dp->dtdo_rtype.dtdt_kind ==
7065 DIF_TYPE_STRING) {
7066 char c = '\0' + 1;
7067 int intuple = act->dta_intuple;
7068 size_t s;
7069
7070 for (s = 0; s < size; s++) {
7071 if (c != '\0')
7072 c = dtrace_load8(val++);
7073
7074 DTRACE_STORE(uint8_t, tomax,
7075 valoffs++, c);
7076
7077 if (c == '\0' && intuple)
7078 break;
7079 }
7080
7081 continue;
7082 }
7083
7084 while (valoffs < end) {
7085 DTRACE_STORE(uint8_t, tomax, valoffs++,
7086 dtrace_load8(val++));
7087 }
7088
7089 continue;
7090 }
7091
7092 switch (size) {
7093 case 0:
7094 break;
7095
7096 case sizeof (uint8_t):
7097 DTRACE_STORE(uint8_t, tomax, valoffs, val);
7098 break;
7099 case sizeof (uint16_t):
7100 DTRACE_STORE(uint16_t, tomax, valoffs, val);
7101 break;
7102 case sizeof (uint32_t):
7103 DTRACE_STORE(uint32_t, tomax, valoffs, val);
7104 break;
7105 case sizeof (uint64_t):
7106 DTRACE_STORE(uint64_t, tomax, valoffs, val);
7107 break;
7108 default:
9186 case DIF_OP_PUSHTV:
9187 if (type != DIF_TYPE_CTF)
9188 err += efunc(pc, "invalid val type %u\n", type);
9189 if (r2 >= nregs)
9190 err += efunc(pc, "invalid register %u\n", r2);
9191 if (rs >= nregs)
9192 err += efunc(pc, "invalid register %u\n", rs);
9193 break;
9194 default:
9195 err += efunc(pc, "invalid opcode %u\n",
9196 DIF_INSTR_OP(instr));
9197 }
9198 }
9199
9200 if (dp->dtdo_len != 0 &&
9201 DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
9202 err += efunc(dp->dtdo_len - 1,
9203 "expected 'ret' as last DIF instruction\n");
9204 }
9205
9206 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
9207 /*
9208 * If we're not returning by reference, the size must be either
9209 * 0 or the size of one of the base types.
9210 */
9211 switch (dp->dtdo_rtype.dtdt_size) {
9212 case 0:
9213 case sizeof (uint8_t):
9214 case sizeof (uint16_t):
9215 case sizeof (uint32_t):
9216 case sizeof (uint64_t):
9217 break;
9218
9219 default:
9220 err += efunc(dp->dtdo_len - 1, "bad return size\n");
9221 }
9222 }
9223
9224 for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
9225 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
9226 dtrace_diftype_t *vt, *et;
|
5702 break;
5703 /*FALLTHROUGH*/
5704 case DIF_OP_LDUH:
5705 regs[rd] = dtrace_load16(regs[r1]);
5706 break;
5707 case DIF_OP_RLDUW:
5708 if (!dtrace_canload(regs[r1], 4, mstate, vstate))
5709 break;
5710 /*FALLTHROUGH*/
5711 case DIF_OP_LDUW:
5712 regs[rd] = dtrace_load32(regs[r1]);
5713 break;
5714 case DIF_OP_RLDX:
5715 if (!dtrace_canload(regs[r1], 8, mstate, vstate))
5716 break;
5717 /*FALLTHROUGH*/
5718 case DIF_OP_LDX:
5719 regs[rd] = dtrace_load64(regs[r1]);
5720 break;
5721 case DIF_OP_ULDSB:
5722 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5723 regs[rd] = (int8_t)
5724 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5725 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5726 break;
5727 case DIF_OP_ULDSH:
5728 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5729 regs[rd] = (int16_t)
5730 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5731 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5732 break;
5733 case DIF_OP_ULDSW:
5734 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5735 regs[rd] = (int32_t)
5736 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5737 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5738 break;
5739 case DIF_OP_ULDUB:
5740 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5741 regs[rd] =
5742 dtrace_fuword8((void *)(uintptr_t)regs[r1]);
5743 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5744 break;
5745 case DIF_OP_ULDUH:
5746 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5747 regs[rd] =
5748 dtrace_fuword16((void *)(uintptr_t)regs[r1]);
5749 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5750 break;
5751 case DIF_OP_ULDUW:
5752 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5753 regs[rd] =
5754 dtrace_fuword32((void *)(uintptr_t)regs[r1]);
5755 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5756 break;
5757 case DIF_OP_ULDX:
5758 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
5759 regs[rd] =
5760 dtrace_fuword64((void *)(uintptr_t)regs[r1]);
5761 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
5762 break;
5763 case DIF_OP_RET:
5764 rval = regs[rd];
5765 pc = textlen;
5766 break;
5767 case DIF_OP_NOP:
5768 break;
5769 case DIF_OP_SETX:
5770 regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
5771 break;
5772 case DIF_OP_SETS:
5773 regs[rd] = (uint64_t)(uintptr_t)
5774 (strtab + DIF_INSTR_STRING(instr));
5775 break;
5776 case DIF_OP_SCMP: {
5777 size_t sz = state->dts_options[DTRACEOPT_STRSIZE];
5778 uintptr_t s1 = regs[r1];
5779 uintptr_t s2 = regs[r2];
5780
5781 if (s1 != NULL &&
6549 }
6550
6551 if (offs >= strsize) {
6552 /*
6553 * If we didn't have room for all of the strings, we don't
6554 * abort processing -- this needn't be a fatal error -- but we
6555 * still want to increment a counter (dts_stkstroverflows) to
6556 * allow this condition to be warned about. (If this is from
6557 * a jstack() action, it is easily tuned via jstackstrsize.)
6558 */
6559 dtrace_error(&state->dts_stkstroverflows);
6560 }
6561
6562 while (offs < strsize)
6563 str[offs++] = '\0';
6564
6565 out:
6566 mstate->dtms_scratch_ptr = old;
6567 }
6568
6569 static void
6570 dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size,
6571 size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind)
6572 {
6573 volatile uint16_t *flags;
6574 uint64_t val = *valp;
6575 size_t valoffs = *valoffsp;
6576
6577 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
6578 ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF);
6579
6580 /*
6581 * If this is a string, we're going to only load until we find the zero
6582 * byte -- after which we'll store zero bytes.
6583 */
6584 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
6585 char c = '\0' + 1;
6586 size_t s;
6587
6588 for (s = 0; s < size; s++) {
6589 if (c != '\0' && dtkind == DIF_TF_BYREF) {
6590 c = dtrace_load8(val++);
6591 } else if (c != '\0' && dtkind == DIF_TF_BYUREF) {
6592 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6593 c = dtrace_fuword8((void *)(uintptr_t)val++);
6594 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6595 if (*flags & CPU_DTRACE_FAULT)
6596 break;
6597 }
6598
6599 DTRACE_STORE(uint8_t, tomax, valoffs++, c);
6600
6601 if (c == '\0' && intuple)
6602 break;
6603 }
6604 } else {
6605 uint8_t c;
6606 while (valoffs < end) {
6607 if (dtkind == DIF_TF_BYREF) {
6608 c = dtrace_load8(val++);
6609 } else if (dtkind == DIF_TF_BYUREF) {
6610 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
6611 c = dtrace_fuword8((void *)(uintptr_t)val++);
6612 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
6613 if (*flags & CPU_DTRACE_FAULT)
6614 break;
6615 }
6616
6617 DTRACE_STORE(uint8_t, tomax,
6618 valoffs++, c);
6619 }
6620 }
6621
6622 *valp = val;
6623 *valoffsp = valoffs;
6624 }
6625
6626 /*
6627 * If you're looking for the epicenter of DTrace, you just found it. This
6628 * is the function called by the provider to fire a probe -- from which all
6629 * subsequent probe-context DTrace activity emanates.
6630 */
6631 void
6632 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
6633 uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
6634 {
6635 processorid_t cpuid;
6636 dtrace_icookie_t cookie;
6637 dtrace_probe_t *probe;
6638 dtrace_mstate_t mstate;
6639 dtrace_ecb_t *ecb;
6640 dtrace_action_t *act;
6641 intptr_t offs;
6642 size_t size;
6643 int vtime, onintr;
6644 volatile uint16_t *flags;
6645 hrtime_t now, end;
7097
7098 if (current == DTRACE_ACTIVITY_COOLDOWN)
7099 break;
7100
7101 if (current != DTRACE_ACTIVITY_WARMUP)
7102 current = DTRACE_ACTIVITY_ACTIVE;
7103
7104 if (dtrace_cas32(activity, current,
7105 DTRACE_ACTIVITY_DRAINING) != current) {
7106 *flags |= CPU_DTRACE_DROP;
7107 continue;
7108 }
7109
7110 break;
7111 }
7112
7113 default:
7114 ASSERT(0);
7115 }
7116
7117 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ||
7118 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) {
7119 uintptr_t end = valoffs + size;
7120
7121 if (tracememsize != 0 &&
7122 valoffs + tracememsize < end) {
7123 end = valoffs + tracememsize;
7124 tracememsize = 0;
7125 }
7126
7127 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF &&
7128 !dtrace_vcanload((void *)(uintptr_t)val,
7129 &dp->dtdo_rtype, &mstate, vstate))
7130 continue;
7131
7132 dtrace_store_by_ref(dp, tomax, size, &valoffs,
7133 &val, end, act->dta_intuple,
7134 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ?
7135 DIF_TF_BYREF: DIF_TF_BYUREF);
7136 continue;
7137 }
7138
7139 switch (size) {
7140 case 0:
7141 break;
7142
7143 case sizeof (uint8_t):
7144 DTRACE_STORE(uint8_t, tomax, valoffs, val);
7145 break;
7146 case sizeof (uint16_t):
7147 DTRACE_STORE(uint16_t, tomax, valoffs, val);
7148 break;
7149 case sizeof (uint32_t):
7150 DTRACE_STORE(uint32_t, tomax, valoffs, val);
7151 break;
7152 case sizeof (uint64_t):
7153 DTRACE_STORE(uint64_t, tomax, valoffs, val);
7154 break;
7155 default:
9233 case DIF_OP_PUSHTV:
9234 if (type != DIF_TYPE_CTF)
9235 err += efunc(pc, "invalid val type %u\n", type);
9236 if (r2 >= nregs)
9237 err += efunc(pc, "invalid register %u\n", r2);
9238 if (rs >= nregs)
9239 err += efunc(pc, "invalid register %u\n", rs);
9240 break;
9241 default:
9242 err += efunc(pc, "invalid opcode %u\n",
9243 DIF_INSTR_OP(instr));
9244 }
9245 }
9246
9247 if (dp->dtdo_len != 0 &&
9248 DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
9249 err += efunc(dp->dtdo_len - 1,
9250 "expected 'ret' as last DIF instruction\n");
9251 }
9252
9253 if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) {
9254 /*
9255 * If we're not returning by reference, the size must be either
9256 * 0 or the size of one of the base types.
9257 */
9258 switch (dp->dtdo_rtype.dtdt_size) {
9259 case 0:
9260 case sizeof (uint8_t):
9261 case sizeof (uint16_t):
9262 case sizeof (uint32_t):
9263 case sizeof (uint64_t):
9264 break;
9265
9266 default:
9267 err += efunc(dp->dtdo_len - 1, "bad return size\n");
9268 }
9269 }
9270
9271 for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
9272 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
9273 dtrace_diftype_t *vt, *et;
|