1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright (c) 2018, Joyent, Inc.
28 * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
29 */
30
31 #include "umem.h"
32
33 #include <sys/vmem_impl_user.h>
34 #include <umem_impl.h>
35
36 #include <alloca.h>
37 #include <limits.h>
38 #include <mdb/mdb_whatis.h>
39 #include <thr_uberdata.h>
40
41 #include "misc.h"
42 #include "leaky.h"
43 #include "dist.h"
44
45 #include "umem_pagesize.h"
46
47 #define UM_ALLOCATED 0x1
48 #define UM_FREE 0x2
49 #define UM_BUFCTL 0x4
50 #define UM_HASH 0x8
51
52 int umem_ready;
53
54 static int umem_stack_depth_warned;
55 static uint32_t umem_max_ncpus;
56 uint32_t umem_stack_depth;
57
58 size_t umem_pagesize;
59
60 #define UMEM_READVAR(var) \
61 (umem_readvar(&(var), #var) == -1 && \
62 (mdb_warn("failed to read "#var), 1))
63
64 int
65 umem_update_variables(void)
66 {
67 size_t pagesize;
68
69 /*
70 * Figure out which type of umem is being used; if it's not there
71 * yet, succeed quietly.
72 */
73 if (umem_set_standalone() == -1) {
74 umem_ready = 0;
75 return (0); /* umem not there yet */
76 }
77
78 /*
79 * Solaris 9 used a different name for umem_max_ncpus. It's
80 * cheap backwards compatibility to check for both names.
81 */
82 if (umem_readvar(&umem_max_ncpus, "umem_max_ncpus") == -1 &&
83 umem_readvar(&umem_max_ncpus, "max_ncpus") == -1) {
84 mdb_warn("unable to read umem_max_ncpus or max_ncpus");
85 return (-1);
86 }
87 if (UMEM_READVAR(umem_ready))
88 return (-1);
89 if (UMEM_READVAR(umem_stack_depth))
90 return (-1);
91 if (UMEM_READVAR(pagesize))
92 return (-1);
93
94 if (umem_stack_depth > UMEM_MAX_STACK_DEPTH) {
95 if (umem_stack_depth_warned == 0) {
96 mdb_warn("umem_stack_depth corrupted (%d > %d)\n",
97 umem_stack_depth, UMEM_MAX_STACK_DEPTH);
98 umem_stack_depth_warned = 1;
99 }
100 umem_stack_depth = 0;
101 }
102
103 umem_pagesize = pagesize;
104
105 return (0);
106 }
107
108 static int
109 umem_ptc_walk_init(mdb_walk_state_t *wsp)
110 {
111 if (wsp->walk_addr == NULL) {
112 if (mdb_layered_walk("ulwp", wsp) == -1) {
113 mdb_warn("couldn't walk 'ulwp'");
114 return (WALK_ERR);
115 }
116 }
117
118 return (WALK_NEXT);
119 }
120
121 static int
122 umem_ptc_walk_step(mdb_walk_state_t *wsp)
123 {
124 uintptr_t this;
125 int rval;
126
127 if (wsp->walk_layer != NULL) {
128 this = (uintptr_t)((ulwp_t *)wsp->walk_layer)->ul_self +
129 (uintptr_t)wsp->walk_arg;
130 } else {
131 this = wsp->walk_addr + (uintptr_t)wsp->walk_arg;
132 }
133
134 for (;;) {
135 if (mdb_vread(&this, sizeof (void *), this) == -1) {
136 mdb_warn("couldn't read ptc buffer at %p", this);
137 return (WALK_ERR);
138 }
139
140 if (this == NULL)
141 break;
142
143 rval = wsp->walk_callback(this, &this, wsp->walk_cbdata);
144
145 if (rval != WALK_NEXT)
146 return (rval);
147 }
148
149 return (wsp->walk_layer != NULL ? WALK_NEXT : WALK_DONE);
150 }
151
152 /*ARGSUSED*/
153 static int
154 umem_init_walkers(uintptr_t addr, const umem_cache_t *c, int *sizes)
155 {
156 mdb_walker_t w;
157 char descr[64];
158 char name[64];
159 int i;
160
161 (void) mdb_snprintf(descr, sizeof (descr),
162 "walk the %s cache", c->cache_name);
163
164 w.walk_name = c->cache_name;
165 w.walk_descr = descr;
166 w.walk_init = umem_walk_init;
167 w.walk_step = umem_walk_step;
168 w.walk_fini = umem_walk_fini;
169 w.walk_init_arg = (void *)addr;
170
171 if (mdb_add_walker(&w) == -1)
172 mdb_warn("failed to add %s walker", c->cache_name);
173
174 if (!(c->cache_flags & UMF_PTC))
175 return (WALK_NEXT);
176
177 /*
178 * For the per-thread cache walker, the address is the offset in the
179 * tm_roots[] array of the ulwp_t.
180 */
181 for (i = 0; sizes[i] != 0; i++) {
182 if (sizes[i] == c->cache_bufsize)
183 break;
184 }
185
186 if (sizes[i] == 0) {
187 mdb_warn("cache %s is cached per-thread, but could not find "
188 "size in umem_alloc_sizes\n", c->cache_name);
189 return (WALK_NEXT);
190 }
191
192 if (i >= NTMEMBASE) {
193 mdb_warn("index for %s (%d) exceeds root slots (%d)\n",
194 c->cache_name, i, NTMEMBASE);
195 return (WALK_NEXT);
196 }
197
198 (void) mdb_snprintf(name, sizeof (name),
199 "umem_ptc_%d", c->cache_bufsize);
200 (void) mdb_snprintf(descr, sizeof (descr),
201 "walk the per-thread cache for %s", c->cache_name);
202
203 w.walk_name = name;
204 w.walk_descr = descr;
205 w.walk_init = umem_ptc_walk_init;
206 w.walk_step = umem_ptc_walk_step;
207 w.walk_fini = NULL;
208 w.walk_init_arg = (void *)offsetof(ulwp_t, ul_tmem.tm_roots[i]);
209
210 if (mdb_add_walker(&w) == -1)
211 mdb_warn("failed to add %s walker", w.walk_name);
212
213 return (WALK_NEXT);
214 }
215
216 /*ARGSUSED*/
217 static void
218 umem_statechange_cb(void *arg)
219 {
220 static int been_ready = 0;
221 GElf_Sym sym;
222 int *sizes;
223
224 #ifndef _KMDB
225 leaky_cleanup(1); /* state changes invalidate leaky state */
226 #endif
227
228 if (umem_update_variables() == -1)
229 return;
230
231 if (been_ready)
232 return;
233
234 if (umem_ready != UMEM_READY)
235 return;
236
237 been_ready = 1;
238
239 /*
240 * In order to determine the tm_roots offset of any cache that is
241 * cached per-thread, we need to have the umem_alloc_sizes array.
242 * Read this, assuring that it is zero-terminated.
243 */
244 if (umem_lookup_by_name("umem_alloc_sizes", &sym) == -1) {
245 mdb_warn("unable to lookup 'umem_alloc_sizes'");
246 return;
247 }
248
249 sizes = mdb_zalloc(sym.st_size + sizeof (int), UM_SLEEP | UM_GC);
250
251 if (mdb_vread(sizes, sym.st_size, (uintptr_t)sym.st_value) == -1) {
252 mdb_warn("couldn't read 'umem_alloc_sizes'");
253 return;
254 }
255
256 (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umem_init_walkers, sizes);
257 }
258
259 int
260 umem_abort_messages(void)
261 {
262 char *umem_error_buffer;
263 uint_t umem_error_begin;
264 GElf_Sym sym;
265 size_t bufsize;
266
267 if (UMEM_READVAR(umem_error_begin))
268 return (DCMD_ERR);
269
270 if (umem_lookup_by_name("umem_error_buffer", &sym) == -1) {
271 mdb_warn("unable to look up umem_error_buffer");
272 return (DCMD_ERR);
273 }
274
275 bufsize = (size_t)sym.st_size;
276
277 umem_error_buffer = mdb_alloc(bufsize+1, UM_SLEEP | UM_GC);
278
279 if (mdb_vread(umem_error_buffer, bufsize, (uintptr_t)sym.st_value)
280 != bufsize) {
281 mdb_warn("unable to read umem_error_buffer");
282 return (DCMD_ERR);
283 }
284 /* put a zero after the end of the buffer to simplify printing */
285 umem_error_buffer[bufsize] = 0;
286
287 if ((umem_error_begin % bufsize) == 0)
288 mdb_printf("%s\n", umem_error_buffer);
289 else {
290 umem_error_buffer[(umem_error_begin % bufsize) - 1] = 0;
291 mdb_printf("%s%s\n",
292 &umem_error_buffer[umem_error_begin % bufsize],
293 umem_error_buffer);
294 }
295
296 return (DCMD_OK);
297 }
298
299 static void
300 umem_log_status(const char *name, umem_log_header_t *val)
301 {
302 umem_log_header_t my_lh;
303 uintptr_t pos = (uintptr_t)val;
304 size_t size;
305
306 if (pos == NULL)
307 return;
308
309 if (mdb_vread(&my_lh, sizeof (umem_log_header_t), pos) == -1) {
310 mdb_warn("\nunable to read umem_%s_log pointer %p",
311 name, pos);
312 return;
313 }
314
315 size = my_lh.lh_chunksize * my_lh.lh_nchunks;
316
317 if (size % (1024 * 1024) == 0)
318 mdb_printf("%s=%dm ", name, size / (1024 * 1024));
319 else if (size % 1024 == 0)
320 mdb_printf("%s=%dk ", name, size / 1024);
321 else
322 mdb_printf("%s=%d ", name, size);
323 }
324
325 typedef struct umem_debug_flags {
326 const char *udf_name;
327 uint_t udf_flags;
328 uint_t udf_clear; /* if 0, uses udf_flags */
329 } umem_debug_flags_t;
330
331 umem_debug_flags_t umem_status_flags[] = {
332 { "random", UMF_RANDOMIZE, UMF_RANDOM },
333 { "default", UMF_AUDIT | UMF_DEADBEEF | UMF_REDZONE | UMF_CONTENTS },
334 { "audit", UMF_AUDIT },
335 { "guards", UMF_DEADBEEF | UMF_REDZONE },
336 { "nosignal", UMF_CHECKSIGNAL },
337 { "firewall", UMF_FIREWALL },
338 { "lite", UMF_LITE },
339 { "checknull", UMF_CHECKNULL },
340 { NULL }
341 };
342
343 /*ARGSUSED*/
344 int
345 umem_status(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *argv)
346 {
347 int umem_logging;
348
349 umem_log_header_t *umem_transaction_log;
350 umem_log_header_t *umem_content_log;
351 umem_log_header_t *umem_failure_log;
352 umem_log_header_t *umem_slab_log;
353
354 mdb_printf("Status:\t\t%s\n",
355 umem_ready == UMEM_READY_INIT_FAILED ? "initialization failed" :
356 umem_ready == UMEM_READY_STARTUP ? "uninitialized" :
357 umem_ready == UMEM_READY_INITING ? "initialization in process" :
358 umem_ready == UMEM_READY ? "ready and active" :
359 umem_ready == 0 ? "not loaded into address space" :
360 "unknown (umem_ready invalid)");
361
362 if (umem_ready == 0)
363 return (DCMD_OK);
364
365 mdb_printf("Concurrency:\t%d\n", umem_max_ncpus);
366
367 if (UMEM_READVAR(umem_logging))
368 goto err;
369 if (UMEM_READVAR(umem_transaction_log))
370 goto err;
371 if (UMEM_READVAR(umem_content_log))
372 goto err;
373 if (UMEM_READVAR(umem_failure_log))
374 goto err;
375 if (UMEM_READVAR(umem_slab_log))
376 goto err;
377
378 mdb_printf("Logs:\t\t");
379 umem_log_status("transaction", umem_transaction_log);
380 umem_log_status("content", umem_content_log);
381 umem_log_status("fail", umem_failure_log);
382 umem_log_status("slab", umem_slab_log);
383 if (!umem_logging)
384 mdb_printf("(inactive)");
385 mdb_printf("\n");
386
387 mdb_printf("Message buffer:\n");
388 return (umem_abort_messages());
389
390 err:
391 mdb_printf("Message buffer:\n");
392 (void) umem_abort_messages();
393 return (DCMD_ERR);
394 }
395
396 typedef struct {
397 uintptr_t ucw_first;
398 uintptr_t ucw_current;
399 } umem_cache_walk_t;
400
401 int
402 umem_cache_walk_init(mdb_walk_state_t *wsp)
403 {
404 umem_cache_walk_t *ucw;
405 umem_cache_t c;
406 uintptr_t cp;
407 GElf_Sym sym;
408
409 if (umem_lookup_by_name("umem_null_cache", &sym) == -1) {
410 mdb_warn("couldn't find umem_null_cache");
411 return (WALK_ERR);
412 }
413
414 cp = (uintptr_t)sym.st_value;
415
416 if (mdb_vread(&c, sizeof (umem_cache_t), cp) == -1) {
417 mdb_warn("couldn't read cache at %p", cp);
418 return (WALK_ERR);
419 }
420
421 ucw = mdb_alloc(sizeof (umem_cache_walk_t), UM_SLEEP);
422
423 ucw->ucw_first = cp;
424 ucw->ucw_current = (uintptr_t)c.cache_next;
425 wsp->walk_data = ucw;
426
427 return (WALK_NEXT);
428 }
429
430 int
431 umem_cache_walk_step(mdb_walk_state_t *wsp)
432 {
433 umem_cache_walk_t *ucw = wsp->walk_data;
434 umem_cache_t c;
435 int status;
436
437 if (mdb_vread(&c, sizeof (umem_cache_t), ucw->ucw_current) == -1) {
438 mdb_warn("couldn't read cache at %p", ucw->ucw_current);
439 return (WALK_DONE);
440 }
441
442 status = wsp->walk_callback(ucw->ucw_current, &c, wsp->walk_cbdata);
443
444 if ((ucw->ucw_current = (uintptr_t)c.cache_next) == ucw->ucw_first)
445 return (WALK_DONE);
446
447 return (status);
448 }
449
450 void
451 umem_cache_walk_fini(mdb_walk_state_t *wsp)
452 {
453 umem_cache_walk_t *ucw = wsp->walk_data;
454 mdb_free(ucw, sizeof (umem_cache_walk_t));
455 }
456
457 typedef struct {
458 umem_cpu_t *ucw_cpus;
459 uint32_t ucw_current;
460 uint32_t ucw_max;
461 } umem_cpu_walk_state_t;
462
463 int
464 umem_cpu_walk_init(mdb_walk_state_t *wsp)
465 {
466 umem_cpu_t *umem_cpus;
467
468 umem_cpu_walk_state_t *ucw;
469
470 if (umem_readvar(&umem_cpus, "umem_cpus") == -1) {
471 mdb_warn("failed to read 'umem_cpus'");
472 return (WALK_ERR);
473 }
474
475 ucw = mdb_alloc(sizeof (*ucw), UM_SLEEP);
476
477 ucw->ucw_cpus = umem_cpus;
478 ucw->ucw_current = 0;
479 ucw->ucw_max = umem_max_ncpus;
480
481 wsp->walk_data = ucw;
482 return (WALK_NEXT);
483 }
484
485 int
486 umem_cpu_walk_step(mdb_walk_state_t *wsp)
487 {
488 umem_cpu_t cpu;
489 umem_cpu_walk_state_t *ucw = wsp->walk_data;
490
491 uintptr_t caddr;
492
493 if (ucw->ucw_current >= ucw->ucw_max)
494 return (WALK_DONE);
495
496 caddr = (uintptr_t)&(ucw->ucw_cpus[ucw->ucw_current]);
497
498 if (mdb_vread(&cpu, sizeof (umem_cpu_t), caddr) == -1) {
499 mdb_warn("failed to read cpu %d", ucw->ucw_current);
500 return (WALK_ERR);
501 }
502
503 ucw->ucw_current++;
504
505 return (wsp->walk_callback(caddr, &cpu, wsp->walk_cbdata));
506 }
507
508 void
509 umem_cpu_walk_fini(mdb_walk_state_t *wsp)
510 {
511 umem_cpu_walk_state_t *ucw = wsp->walk_data;
512
513 mdb_free(ucw, sizeof (*ucw));
514 }
515
516 int
517 umem_cpu_cache_walk_init(mdb_walk_state_t *wsp)
518 {
519 if (wsp->walk_addr == NULL) {
520 mdb_warn("umem_cpu_cache doesn't support global walks");
521 return (WALK_ERR);
522 }
523
524 if (mdb_layered_walk("umem_cpu", wsp) == -1) {
525 mdb_warn("couldn't walk 'umem_cpu'");
526 return (WALK_ERR);
527 }
528
529 wsp->walk_data = (void *)wsp->walk_addr;
530
531 return (WALK_NEXT);
532 }
533
534 int
535 umem_cpu_cache_walk_step(mdb_walk_state_t *wsp)
536 {
537 uintptr_t caddr = (uintptr_t)wsp->walk_data;
538 const umem_cpu_t *cpu = wsp->walk_layer;
539 umem_cpu_cache_t cc;
540
541 caddr += cpu->cpu_cache_offset;
542
543 if (mdb_vread(&cc, sizeof (umem_cpu_cache_t), caddr) == -1) {
544 mdb_warn("couldn't read umem_cpu_cache at %p", caddr);
545 return (WALK_ERR);
546 }
547
548 return (wsp->walk_callback(caddr, &cc, wsp->walk_cbdata));
549 }
550
551 int
552 umem_slab_walk_init(mdb_walk_state_t *wsp)
553 {
554 uintptr_t caddr = wsp->walk_addr;
555 umem_cache_t c;
556
557 if (caddr == NULL) {
558 mdb_warn("umem_slab doesn't support global walks\n");
559 return (WALK_ERR);
560 }
561
562 if (mdb_vread(&c, sizeof (c), caddr) == -1) {
563 mdb_warn("couldn't read umem_cache at %p", caddr);
564 return (WALK_ERR);
565 }
566
567 wsp->walk_data =
568 (void *)(caddr + offsetof(umem_cache_t, cache_nullslab));
569 wsp->walk_addr = (uintptr_t)c.cache_nullslab.slab_next;
570
571 return (WALK_NEXT);
572 }
573
574 int
575 umem_slab_walk_partial_init(mdb_walk_state_t *wsp)
576 {
577 uintptr_t caddr = wsp->walk_addr;
578 umem_cache_t c;
579
580 if (caddr == NULL) {
581 mdb_warn("umem_slab_partial doesn't support global walks\n");
582 return (WALK_ERR);
583 }
584
585 if (mdb_vread(&c, sizeof (c), caddr) == -1) {
586 mdb_warn("couldn't read umem_cache at %p", caddr);
587 return (WALK_ERR);
588 }
589
590 wsp->walk_data =
591 (void *)(caddr + offsetof(umem_cache_t, cache_nullslab));
592 wsp->walk_addr = (uintptr_t)c.cache_freelist;
593
594 /*
595 * Some consumers (umem_walk_step(), in particular) require at
596 * least one callback if there are any buffers in the cache. So
597 * if there are *no* partial slabs, report the last full slab, if
598 * any.
599 *
600 * Yes, this is ugly, but it's cleaner than the other possibilities.
601 */
602 if ((uintptr_t)wsp->walk_data == wsp->walk_addr)
603 wsp->walk_addr = (uintptr_t)c.cache_nullslab.slab_prev;
604
605 return (WALK_NEXT);
606 }
607
608 int
609 umem_slab_walk_step(mdb_walk_state_t *wsp)
610 {
611 umem_slab_t s;
612 uintptr_t addr = wsp->walk_addr;
613 uintptr_t saddr = (uintptr_t)wsp->walk_data;
614 uintptr_t caddr = saddr - offsetof(umem_cache_t, cache_nullslab);
615
616 if (addr == saddr)
617 return (WALK_DONE);
618
619 if (mdb_vread(&s, sizeof (s), addr) == -1) {
620 mdb_warn("failed to read slab at %p", wsp->walk_addr);
621 return (WALK_ERR);
622 }
623
624 if ((uintptr_t)s.slab_cache != caddr) {
625 mdb_warn("slab %p isn't in cache %p (in cache %p)\n",
626 addr, caddr, s.slab_cache);
627 return (WALK_ERR);
628 }
629
630 wsp->walk_addr = (uintptr_t)s.slab_next;
631
632 return (wsp->walk_callback(addr, &s, wsp->walk_cbdata));
633 }
634
635 int
636 umem_cache(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *argv)
637 {
638 umem_cache_t c;
639
640 if (!(flags & DCMD_ADDRSPEC)) {
641 if (mdb_walk_dcmd("umem_cache", "umem_cache", ac, argv) == -1) {
642 mdb_warn("can't walk umem_cache");
643 return (DCMD_ERR);
644 }
645 return (DCMD_OK);
646 }
647
648 if (DCMD_HDRSPEC(flags))
649 mdb_printf("%-?s %-25s %4s %8s %8s %8s\n", "ADDR", "NAME",
650 "FLAG", "CFLAG", "BUFSIZE", "BUFTOTL");
651
652 if (mdb_vread(&c, sizeof (c), addr) == -1) {
653 mdb_warn("couldn't read umem_cache at %p", addr);
654 return (DCMD_ERR);
655 }
656
657 mdb_printf("%0?p %-25s %04x %08x %8ld %8lld\n", addr, c.cache_name,
658 c.cache_flags, c.cache_cflags, c.cache_bufsize, c.cache_buftotal);
659
660 return (DCMD_OK);
661 }
662
663 static int
664 addrcmp(const void *lhs, const void *rhs)
665 {
666 uintptr_t p1 = *((uintptr_t *)lhs);
667 uintptr_t p2 = *((uintptr_t *)rhs);
668
669 if (p1 < p2)
670 return (-1);
671 if (p1 > p2)
672 return (1);
673 return (0);
674 }
675
676 static int
677 bufctlcmp(const umem_bufctl_audit_t **lhs, const umem_bufctl_audit_t **rhs)
678 {
679 const umem_bufctl_audit_t *bcp1 = *lhs;
680 const umem_bufctl_audit_t *bcp2 = *rhs;
681
682 if (bcp1->bc_timestamp > bcp2->bc_timestamp)
683 return (-1);
684
685 if (bcp1->bc_timestamp < bcp2->bc_timestamp)
686 return (1);
687
688 return (0);
689 }
690
691 typedef struct umem_hash_walk {
692 uintptr_t *umhw_table;
693 size_t umhw_nelems;
694 size_t umhw_pos;
695 umem_bufctl_t umhw_cur;
696 } umem_hash_walk_t;
697
698 int
699 umem_hash_walk_init(mdb_walk_state_t *wsp)
700 {
701 umem_hash_walk_t *umhw;
702 uintptr_t *hash;
703 umem_cache_t c;
704 uintptr_t haddr, addr = wsp->walk_addr;
705 size_t nelems;
706 size_t hsize;
707
708 if (addr == NULL) {
709 mdb_warn("umem_hash doesn't support global walks\n");
710 return (WALK_ERR);
711 }
712
713 if (mdb_vread(&c, sizeof (c), addr) == -1) {
714 mdb_warn("couldn't read cache at addr %p", addr);
715 return (WALK_ERR);
716 }
717
718 if (!(c.cache_flags & UMF_HASH)) {
719 mdb_warn("cache %p doesn't have a hash table\n", addr);
720 return (WALK_DONE); /* nothing to do */
721 }
722
723 umhw = mdb_zalloc(sizeof (umem_hash_walk_t), UM_SLEEP);
724 umhw->umhw_cur.bc_next = NULL;
725 umhw->umhw_pos = 0;
726
727 umhw->umhw_nelems = nelems = c.cache_hash_mask + 1;
728 hsize = nelems * sizeof (uintptr_t);
729 haddr = (uintptr_t)c.cache_hash_table;
730
731 umhw->umhw_table = hash = mdb_alloc(hsize, UM_SLEEP);
732 if (mdb_vread(hash, hsize, haddr) == -1) {
733 mdb_warn("failed to read hash table at %p", haddr);
734 mdb_free(hash, hsize);
735 mdb_free(umhw, sizeof (umem_hash_walk_t));
736 return (WALK_ERR);
737 }
738
739 wsp->walk_data = umhw;
740
741 return (WALK_NEXT);
742 }
743
744 int
745 umem_hash_walk_step(mdb_walk_state_t *wsp)
746 {
747 umem_hash_walk_t *umhw = wsp->walk_data;
748 uintptr_t addr = NULL;
749
750 if ((addr = (uintptr_t)umhw->umhw_cur.bc_next) == NULL) {
751 while (umhw->umhw_pos < umhw->umhw_nelems) {
752 if ((addr = umhw->umhw_table[umhw->umhw_pos++]) != NULL)
753 break;
754 }
755 }
756 if (addr == NULL)
757 return (WALK_DONE);
758
759 if (mdb_vread(&umhw->umhw_cur, sizeof (umem_bufctl_t), addr) == -1) {
760 mdb_warn("couldn't read umem_bufctl_t at addr %p", addr);
761 return (WALK_ERR);
762 }
763
764 return (wsp->walk_callback(addr, &umhw->umhw_cur, wsp->walk_cbdata));
765 }
766
767 void
768 umem_hash_walk_fini(mdb_walk_state_t *wsp)
769 {
770 umem_hash_walk_t *umhw = wsp->walk_data;
771
772 if (umhw == NULL)
773 return;
774
775 mdb_free(umhw->umhw_table, umhw->umhw_nelems * sizeof (uintptr_t));
776 mdb_free(umhw, sizeof (umem_hash_walk_t));
777 }
778
779 /*
780 * Find the address of the bufctl structure for the address 'buf' in cache
781 * 'cp', which is at address caddr, and place it in *out.
782 */
783 static int
784 umem_hash_lookup(umem_cache_t *cp, uintptr_t caddr, void *buf, uintptr_t *out)
785 {
786 uintptr_t bucket = (uintptr_t)UMEM_HASH(cp, buf);
787 umem_bufctl_t *bcp;
788 umem_bufctl_t bc;
789
790 if (mdb_vread(&bcp, sizeof (umem_bufctl_t *), bucket) == -1) {
791 mdb_warn("unable to read hash bucket for %p in cache %p",
792 buf, caddr);
793 return (-1);
794 }
795
796 while (bcp != NULL) {
797 if (mdb_vread(&bc, sizeof (umem_bufctl_t),
798 (uintptr_t)bcp) == -1) {
799 mdb_warn("unable to read bufctl at %p", bcp);
800 return (-1);
801 }
802 if (bc.bc_addr == buf) {
803 *out = (uintptr_t)bcp;
804 return (0);
805 }
806 bcp = bc.bc_next;
807 }
808
809 mdb_warn("unable to find bufctl for %p in cache %p\n", buf, caddr);
810 return (-1);
811 }
812
813 int
814 umem_get_magsize(const umem_cache_t *cp)
815 {
816 uintptr_t addr = (uintptr_t)cp->cache_magtype;
817 GElf_Sym mt_sym;
818 umem_magtype_t mt;
819 int res;
820
821 /*
822 * if cpu 0 has a non-zero magsize, it must be correct. caches
823 * with UMF_NOMAGAZINE have disabled their magazine layers, so
824 * it is okay to return 0 for them.
825 */
826 if ((res = cp->cache_cpu[0].cc_magsize) != 0 ||
827 (cp->cache_flags & UMF_NOMAGAZINE))
828 return (res);
829
830 if (umem_lookup_by_name("umem_magtype", &mt_sym) == -1) {
831 mdb_warn("unable to read 'umem_magtype'");
832 } else if (addr < mt_sym.st_value ||
833 addr + sizeof (mt) - 1 > mt_sym.st_value + mt_sym.st_size - 1 ||
834 ((addr - mt_sym.st_value) % sizeof (mt)) != 0) {
835 mdb_warn("cache '%s' has invalid magtype pointer (%p)\n",
836 cp->cache_name, addr);
837 return (0);
838 }
839 if (mdb_vread(&mt, sizeof (mt), addr) == -1) {
840 mdb_warn("unable to read magtype at %a", addr);
841 return (0);
842 }
843 return (mt.mt_magsize);
844 }
845
846 /*ARGSUSED*/
847 static int
848 umem_estimate_slab(uintptr_t addr, const umem_slab_t *sp, size_t *est)
849 {
850 *est -= (sp->slab_chunks - sp->slab_refcnt);
851
852 return (WALK_NEXT);
853 }
854
855 /*
856 * Returns an upper bound on the number of allocated buffers in a given
857 * cache.
858 */
859 size_t
860 umem_estimate_allocated(uintptr_t addr, const umem_cache_t *cp)
861 {
862 int magsize;
863 size_t cache_est;
864
865 cache_est = cp->cache_buftotal;
866
867 (void) mdb_pwalk("umem_slab_partial",
868 (mdb_walk_cb_t)umem_estimate_slab, &cache_est, addr);
869
870 if ((magsize = umem_get_magsize(cp)) != 0) {
871 size_t mag_est = cp->cache_full.ml_total * magsize;
872
873 if (cache_est >= mag_est) {
874 cache_est -= mag_est;
875 } else {
876 mdb_warn("cache %p's magazine layer holds more buffers "
877 "than the slab layer.\n", addr);
878 }
879 }
880 return (cache_est);
881 }
882
883 #define READMAG_ROUNDS(rounds) { \
884 if (mdb_vread(mp, magbsize, (uintptr_t)ump) == -1) { \
885 mdb_warn("couldn't read magazine at %p", ump); \
886 goto fail; \
887 } \
888 for (i = 0; i < rounds; i++) { \
889 maglist[magcnt++] = mp->mag_round[i]; \
890 if (magcnt == magmax) { \
891 mdb_warn("%d magazines exceeds fudge factor\n", \
892 magcnt); \
893 goto fail; \
894 } \
895 } \
896 }
897
898 static int
899 umem_read_magazines(umem_cache_t *cp, uintptr_t addr,
900 void ***maglistp, size_t *magcntp, size_t *magmaxp)
901 {
902 umem_magazine_t *ump, *mp;
903 void **maglist = NULL;
904 int i, cpu;
905 size_t magsize, magmax, magbsize;
906 size_t magcnt = 0;
907
908 /*
909 * Read the magtype out of the cache, after verifying the pointer's
910 * correctness.
911 */
912 magsize = umem_get_magsize(cp);
913 if (magsize == 0) {
914 *maglistp = NULL;
915 *magcntp = 0;
916 *magmaxp = 0;
917 return (0);
918 }
919
920 /*
921 * There are several places where we need to go buffer hunting:
922 * the per-CPU loaded magazine, the per-CPU spare full magazine,
923 * and the full magazine list in the depot.
924 *
925 * For an upper bound on the number of buffers in the magazine
926 * layer, we have the number of magazines on the cache_full
927 * list plus at most two magazines per CPU (the loaded and the
928 * spare). Toss in 100 magazines as a fudge factor in case this
929 * is live (the number "100" comes from the same fudge factor in
930 * crash(1M)).
931 */
932 magmax = (cp->cache_full.ml_total + 2 * umem_max_ncpus + 100) * magsize;
933 magbsize = offsetof(umem_magazine_t, mag_round[magsize]);
934
935 if (magbsize >= PAGESIZE / 2) {
936 mdb_warn("magazine size for cache %p unreasonable (%x)\n",
937 addr, magbsize);
938 return (-1);
939 }
940
941 maglist = mdb_alloc(magmax * sizeof (void *), UM_SLEEP);
942 mp = mdb_alloc(magbsize, UM_SLEEP);
943 if (mp == NULL || maglist == NULL)
944 goto fail;
945
946 /*
947 * First up: the magazines in the depot (i.e. on the cache_full list).
948 */
949 for (ump = cp->cache_full.ml_list; ump != NULL; ) {
950 READMAG_ROUNDS(magsize);
951 ump = mp->mag_next;
952
953 if (ump == cp->cache_full.ml_list)
954 break; /* cache_full list loop detected */
955 }
956
957 dprintf(("cache_full list done\n"));
958
959 /*
960 * Now whip through the CPUs, snagging the loaded magazines
961 * and full spares.
962 */
963 for (cpu = 0; cpu < umem_max_ncpus; cpu++) {
964 umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu];
965
966 dprintf(("reading cpu cache %p\n",
967 (uintptr_t)ccp - (uintptr_t)cp + addr));
968
969 if (ccp->cc_rounds > 0 &&
970 (ump = ccp->cc_loaded) != NULL) {
971 dprintf(("reading %d loaded rounds\n", ccp->cc_rounds));
972 READMAG_ROUNDS(ccp->cc_rounds);
973 }
974
975 if (ccp->cc_prounds > 0 &&
976 (ump = ccp->cc_ploaded) != NULL) {
977 dprintf(("reading %d previously loaded rounds\n",
978 ccp->cc_prounds));
979 READMAG_ROUNDS(ccp->cc_prounds);
980 }
981 }
982
983 dprintf(("magazine layer: %d buffers\n", magcnt));
984
985 mdb_free(mp, magbsize);
986
987 *maglistp = maglist;
988 *magcntp = magcnt;
989 *magmaxp = magmax;
990
991 return (0);
992
993 fail:
994 if (mp)
995 mdb_free(mp, magbsize);
996 if (maglist)
997 mdb_free(maglist, magmax * sizeof (void *));
998
999 return (-1);
1000 }
1001
1002 typedef struct umem_read_ptc_walk {
1003 void **urpw_buf;
1004 size_t urpw_cnt;
1005 size_t urpw_max;
1006 } umem_read_ptc_walk_t;
1007
1008 /*ARGSUSED*/
1009 static int
1010 umem_read_ptc_walk_buf(uintptr_t addr,
1011 const void *ignored, umem_read_ptc_walk_t *urpw)
1012 {
1013 if (urpw->urpw_cnt == urpw->urpw_max) {
1014 size_t nmax = urpw->urpw_max ? (urpw->urpw_max << 1) : 1;
1015 void **new = mdb_zalloc(nmax * sizeof (void *), UM_SLEEP);
1016
1017 if (nmax > 1) {
1018 size_t osize = urpw->urpw_max * sizeof (void *);
1019 bcopy(urpw->urpw_buf, new, osize);
1020 mdb_free(urpw->urpw_buf, osize);
1021 }
1022
1023 urpw->urpw_buf = new;
1024 urpw->urpw_max = nmax;
1025 }
1026
1027 urpw->urpw_buf[urpw->urpw_cnt++] = (void *)addr;
1028
1029 return (WALK_NEXT);
1030 }
1031
1032 static int
1033 umem_read_ptc(umem_cache_t *cp,
1034 void ***buflistp, size_t *bufcntp, size_t *bufmaxp)
1035 {
1036 umem_read_ptc_walk_t urpw;
1037 char walk[60];
1038 int rval;
1039
1040 if (!(cp->cache_flags & UMF_PTC))
1041 return (0);
1042
1043 (void) mdb_snprintf(walk, sizeof (walk), "umem_ptc_%d",
1044 cp->cache_bufsize);
1045
1046 urpw.urpw_buf = *buflistp;
1047 urpw.urpw_cnt = *bufcntp;
1048 urpw.urpw_max = *bufmaxp;
1049
1050 if ((rval = mdb_walk(walk,
1051 (mdb_walk_cb_t)umem_read_ptc_walk_buf, &urpw)) == -1) {
1052 mdb_warn("couldn't walk %s", walk);
1053 }
1054
1055 *buflistp = urpw.urpw_buf;
1056 *bufcntp = urpw.urpw_cnt;
1057 *bufmaxp = urpw.urpw_max;
1058
1059 return (rval);
1060 }
1061
1062 static int
1063 umem_walk_callback(mdb_walk_state_t *wsp, uintptr_t buf)
1064 {
1065 return (wsp->walk_callback(buf, NULL, wsp->walk_cbdata));
1066 }
1067
1068 static int
1069 bufctl_walk_callback(umem_cache_t *cp, mdb_walk_state_t *wsp, uintptr_t buf)
1070 {
1071 umem_bufctl_audit_t *b;
1072 UMEM_LOCAL_BUFCTL_AUDIT(&b);
1073
1074 /*
1075 * if UMF_AUDIT is not set, we know that we're looking at a
1076 * umem_bufctl_t.
1077 */
1078 if (!(cp->cache_flags & UMF_AUDIT) ||
1079 mdb_vread(b, UMEM_BUFCTL_AUDIT_SIZE, buf) == -1) {
1080 (void) memset(b, 0, UMEM_BUFCTL_AUDIT_SIZE);
1081 if (mdb_vread(b, sizeof (umem_bufctl_t), buf) == -1) {
1082 mdb_warn("unable to read bufctl at %p", buf);
1083 return (WALK_ERR);
1084 }
1085 }
1086
1087 return (wsp->walk_callback(buf, b, wsp->walk_cbdata));
1088 }
1089
1090 typedef struct umem_walk {
1091 int umw_type;
1092
1093 uintptr_t umw_addr; /* cache address */
1094 umem_cache_t *umw_cp;
1095 size_t umw_csize;
1096
1097 /*
1098 * magazine layer
1099 */
1100 void **umw_maglist;
1101 size_t umw_max;
1102 size_t umw_count;
1103 size_t umw_pos;
1104
1105 /*
1106 * slab layer
1107 */
1108 char *umw_valid; /* to keep track of freed buffers */
1109 char *umw_ubase; /* buffer for slab data */
1110 } umem_walk_t;
1111
1112 static int
1113 umem_walk_init_common(mdb_walk_state_t *wsp, int type)
1114 {
1115 umem_walk_t *umw;
1116 int csize;
1117 umem_cache_t *cp;
1118 size_t vm_quantum;
1119
1120 size_t magmax, magcnt;
1121 void **maglist = NULL;
1122 uint_t chunksize, slabsize;
1123 int status = WALK_ERR;
1124 uintptr_t addr = wsp->walk_addr;
1125 const char *layered;
1126
1127 type &= ~UM_HASH;
1128
1129 if (addr == NULL) {
1130 mdb_warn("umem walk doesn't support global walks\n");
1131 return (WALK_ERR);
1132 }
1133
1134 dprintf(("walking %p\n", addr));
1135
1136 /*
1137 * The number of "cpus" determines how large the cache is.
1138 */
1139 csize = UMEM_CACHE_SIZE(umem_max_ncpus);
1140 cp = mdb_alloc(csize, UM_SLEEP);
1141
1142 if (mdb_vread(cp, csize, addr) == -1) {
1143 mdb_warn("couldn't read cache at addr %p", addr);
1144 goto out2;
1145 }
1146
1147 /*
1148 * It's easy for someone to hand us an invalid cache address.
1149 * Unfortunately, it is hard for this walker to survive an
1150 * invalid cache cleanly. So we make sure that:
1151 *
1152 * 1. the vmem arena for the cache is readable,
1153 * 2. the vmem arena's quantum is a power of 2,
1154 * 3. our slabsize is a multiple of the quantum, and
1155 * 4. our chunksize is >0 and less than our slabsize.
1156 */
1157 if (mdb_vread(&vm_quantum, sizeof (vm_quantum),
1158 (uintptr_t)&cp->cache_arena->vm_quantum) == -1 ||
1159 vm_quantum == 0 ||
1160 (vm_quantum & (vm_quantum - 1)) != 0 ||
1161 cp->cache_slabsize < vm_quantum ||
1162 P2PHASE(cp->cache_slabsize, vm_quantum) != 0 ||
1163 cp->cache_chunksize == 0 ||
1164 cp->cache_chunksize > cp->cache_slabsize) {
1165 mdb_warn("%p is not a valid umem_cache_t\n", addr);
1166 goto out2;
1167 }
1168
1169 dprintf(("buf total is %d\n", cp->cache_buftotal));
1170
1171 if (cp->cache_buftotal == 0) {
1172 mdb_free(cp, csize);
1173 return (WALK_DONE);
1174 }
1175
1176 /*
1177 * If they ask for bufctls, but it's a small-slab cache,
1178 * there is nothing to report.
1179 */
1180 if ((type & UM_BUFCTL) && !(cp->cache_flags & UMF_HASH)) {
1181 dprintf(("bufctl requested, not UMF_HASH (flags: %p)\n",
1182 cp->cache_flags));
1183 mdb_free(cp, csize);
1184 return (WALK_DONE);
1185 }
1186
1187 /*
1188 * Read in the contents of the magazine layer
1189 */
1190 if (umem_read_magazines(cp, addr, &maglist, &magcnt, &magmax) != 0)
1191 goto out2;
1192
1193 /*
1194 * Read in the contents of the per-thread caches, if any
1195 */
1196 if (umem_read_ptc(cp, &maglist, &magcnt, &magmax) != 0)
1197 goto out2;
1198
1199 /*
1200 * We have all of the buffers from the magazines and from the
1201 * per-thread cache (if any); if we are walking allocated buffers,
1202 * sort them so we can bsearch them later.
1203 */
1204 if (type & UM_ALLOCATED)
1205 qsort(maglist, magcnt, sizeof (void *), addrcmp);
1206
1207 wsp->walk_data = umw = mdb_zalloc(sizeof (umem_walk_t), UM_SLEEP);
1208
1209 umw->umw_type = type;
1210 umw->umw_addr = addr;
1211 umw->umw_cp = cp;
1212 umw->umw_csize = csize;
1213 umw->umw_maglist = maglist;
1214 umw->umw_max = magmax;
1215 umw->umw_count = magcnt;
1216 umw->umw_pos = 0;
1217
1218 /*
1219 * When walking allocated buffers in a UMF_HASH cache, we walk the
1220 * hash table instead of the slab layer.
1221 */
1222 if ((cp->cache_flags & UMF_HASH) && (type & UM_ALLOCATED)) {
1223 layered = "umem_hash";
1224
1225 umw->umw_type |= UM_HASH;
1226 } else {
1227 /*
1228 * If we are walking freed buffers, we only need the
1229 * magazine layer plus the partially allocated slabs.
1230 * To walk allocated buffers, we need all of the slabs.
1231 */
1232 if (type & UM_ALLOCATED)
1233 layered = "umem_slab";
1234 else
1235 layered = "umem_slab_partial";
1236
1237 /*
1238 * for small-slab caches, we read in the entire slab. For
1239 * freed buffers, we can just walk the freelist. For
1240 * allocated buffers, we use a 'valid' array to track
1241 * the freed buffers.
1242 */
1243 if (!(cp->cache_flags & UMF_HASH)) {
1244 chunksize = cp->cache_chunksize;
1245 slabsize = cp->cache_slabsize;
1246
1247 umw->umw_ubase = mdb_alloc(slabsize +
1248 sizeof (umem_bufctl_t), UM_SLEEP);
1249
1250 if (type & UM_ALLOCATED)
1251 umw->umw_valid =
1252 mdb_alloc(slabsize / chunksize, UM_SLEEP);
1253 }
1254 }
1255
1256 status = WALK_NEXT;
1257
1258 if (mdb_layered_walk(layered, wsp) == -1) {
1259 mdb_warn("unable to start layered '%s' walk", layered);
1260 status = WALK_ERR;
1261 }
1262
1263 out1:
1264 if (status == WALK_ERR) {
1265 if (umw->umw_valid)
1266 mdb_free(umw->umw_valid, slabsize / chunksize);
1267
1268 if (umw->umw_ubase)
1269 mdb_free(umw->umw_ubase, slabsize +
1270 sizeof (umem_bufctl_t));
1271
1272 if (umw->umw_maglist)
1273 mdb_free(umw->umw_maglist, umw->umw_max *
1274 sizeof (uintptr_t));
1275
1276 mdb_free(umw, sizeof (umem_walk_t));
1277 wsp->walk_data = NULL;
1278 }
1279
1280 out2:
1281 if (status == WALK_ERR)
1282 mdb_free(cp, csize);
1283
1284 return (status);
1285 }
1286
1287 int
1288 umem_walk_step(mdb_walk_state_t *wsp)
1289 {
1290 umem_walk_t *umw = wsp->walk_data;
1291 int type = umw->umw_type;
1292 umem_cache_t *cp = umw->umw_cp;
1293
1294 void **maglist = umw->umw_maglist;
1295 int magcnt = umw->umw_count;
1296
1297 uintptr_t chunksize, slabsize;
1298 uintptr_t addr;
1299 const umem_slab_t *sp;
1300 const umem_bufctl_t *bcp;
1301 umem_bufctl_t bc;
1302
1303 int chunks;
1304 char *kbase;
1305 void *buf;
1306 int i, ret;
1307
1308 char *valid, *ubase;
1309
1310 /*
1311 * first, handle the 'umem_hash' layered walk case
1312 */
1313 if (type & UM_HASH) {
1314 /*
1315 * We have a buffer which has been allocated out of the
1316 * global layer. We need to make sure that it's not
1317 * actually sitting in a magazine before we report it as
1318 * an allocated buffer.
1319 */
1320 buf = ((const umem_bufctl_t *)wsp->walk_layer)->bc_addr;
1321
1322 if (magcnt > 0 &&
1323 bsearch(&buf, maglist, magcnt, sizeof (void *),
1324 addrcmp) != NULL)
1325 return (WALK_NEXT);
1326
1327 if (type & UM_BUFCTL)
1328 return (bufctl_walk_callback(cp, wsp, wsp->walk_addr));
1329
1330 return (umem_walk_callback(wsp, (uintptr_t)buf));
1331 }
1332
1333 ret = WALK_NEXT;
1334
1335 addr = umw->umw_addr;
1336
1337 /*
1338 * If we're walking freed buffers, report everything in the
1339 * magazine layer before processing the first slab.
1340 */
1341 if ((type & UM_FREE) && magcnt != 0) {
1342 umw->umw_count = 0; /* only do this once */
1343 for (i = 0; i < magcnt; i++) {
1344 buf = maglist[i];
1345
1346 if (type & UM_BUFCTL) {
1347 uintptr_t out;
1348
1349 if (cp->cache_flags & UMF_BUFTAG) {
1350 umem_buftag_t *btp;
1351 umem_buftag_t tag;
1352
1353 /* LINTED - alignment */
1354 btp = UMEM_BUFTAG(cp, buf);
1355 if (mdb_vread(&tag, sizeof (tag),
1356 (uintptr_t)btp) == -1) {
1357 mdb_warn("reading buftag for "
1358 "%p at %p", buf, btp);
1359 continue;
1360 }
1361 out = (uintptr_t)tag.bt_bufctl;
1362 } else {
1363 if (umem_hash_lookup(cp, addr, buf,
1364 &out) == -1)
1365 continue;
1366 }
1367 ret = bufctl_walk_callback(cp, wsp, out);
1368 } else {
1369 ret = umem_walk_callback(wsp, (uintptr_t)buf);
1370 }
1371
1372 if (ret != WALK_NEXT)
1373 return (ret);
1374 }
1375 }
1376
1377 /*
1378 * Handle the buffers in the current slab
1379 */
1380 chunksize = cp->cache_chunksize;
1381 slabsize = cp->cache_slabsize;
1382
1383 sp = wsp->walk_layer;
1384 chunks = sp->slab_chunks;
1385 kbase = sp->slab_base;
1386
1387 dprintf(("kbase is %p\n", kbase));
1388
1389 if (!(cp->cache_flags & UMF_HASH)) {
1390 valid = umw->umw_valid;
1391 ubase = umw->umw_ubase;
1392
1393 if (mdb_vread(ubase, chunks * chunksize,
1394 (uintptr_t)kbase) == -1) {
1395 mdb_warn("failed to read slab contents at %p", kbase);
1396 return (WALK_ERR);
1397 }
1398
1399 /*
1400 * Set up the valid map as fully allocated -- we'll punch
1401 * out the freelist.
1402 */
1403 if (type & UM_ALLOCATED)
1404 (void) memset(valid, 1, chunks);
1405 } else {
1406 valid = NULL;
1407 ubase = NULL;
1408 }
1409
1410 /*
1411 * walk the slab's freelist
1412 */
1413 bcp = sp->slab_head;
1414
1415 dprintf(("refcnt is %d; chunks is %d\n", sp->slab_refcnt, chunks));
1416
1417 /*
1418 * since we could be in the middle of allocating a buffer,
1419 * our refcnt could be one higher than it aught. So we
1420 * check one further on the freelist than the count allows.
1421 */
1422 for (i = sp->slab_refcnt; i <= chunks; i++) {
1423 uint_t ndx;
1424
1425 dprintf(("bcp is %p\n", bcp));
1426
1427 if (bcp == NULL) {
1428 if (i == chunks)
1429 break;
1430 mdb_warn(
1431 "slab %p in cache %p freelist too short by %d\n",
1432 sp, addr, chunks - i);
1433 break;
1434 }
1435
1436 if (cp->cache_flags & UMF_HASH) {
1437 if (mdb_vread(&bc, sizeof (bc), (uintptr_t)bcp) == -1) {
1438 mdb_warn("failed to read bufctl ptr at %p",
1439 bcp);
1440 break;
1441 }
1442 buf = bc.bc_addr;
1443 } else {
1444 /*
1445 * Otherwise the buffer is (or should be) in the slab
1446 * that we've read in; determine its offset in the
1447 * slab, validate that it's not corrupt, and add to
1448 * our base address to find the umem_bufctl_t. (Note
1449 * that we don't need to add the size of the bufctl
1450 * to our offset calculation because of the slop that's
1451 * allocated for the buffer at ubase.)
1452 */
1453 uintptr_t offs = (uintptr_t)bcp - (uintptr_t)kbase;
1454
1455 if (offs > chunks * chunksize) {
1456 mdb_warn("found corrupt bufctl ptr %p"
1457 " in slab %p in cache %p\n", bcp,
1458 wsp->walk_addr, addr);
1459 break;
1460 }
1461
1462 bc = *((umem_bufctl_t *)((uintptr_t)ubase + offs));
1463 buf = UMEM_BUF(cp, bcp);
1464 }
1465
1466 ndx = ((uintptr_t)buf - (uintptr_t)kbase) / chunksize;
1467
1468 if (ndx > slabsize / cp->cache_bufsize) {
1469 /*
1470 * This is very wrong; we have managed to find
1471 * a buffer in the slab which shouldn't
1472 * actually be here. Emit a warning, and
1473 * try to continue.
1474 */
1475 mdb_warn("buf %p is out of range for "
1476 "slab %p, cache %p\n", buf, sp, addr);
1477 } else if (type & UM_ALLOCATED) {
1478 /*
1479 * we have found a buffer on the slab's freelist;
1480 * clear its entry
1481 */
1482 valid[ndx] = 0;
1483 } else {
1484 /*
1485 * Report this freed buffer
1486 */
1487 if (type & UM_BUFCTL) {
1488 ret = bufctl_walk_callback(cp, wsp,
1489 (uintptr_t)bcp);
1490 } else {
1491 ret = umem_walk_callback(wsp, (uintptr_t)buf);
1492 }
1493 if (ret != WALK_NEXT)
1494 return (ret);
1495 }
1496
1497 bcp = bc.bc_next;
1498 }
1499
1500 if (bcp != NULL) {
1501 dprintf(("slab %p in cache %p freelist too long (%p)\n",
1502 sp, addr, bcp));
1503 }
1504
1505 /*
1506 * If we are walking freed buffers, the loop above handled reporting
1507 * them.
1508 */
1509 if (type & UM_FREE)
1510 return (WALK_NEXT);
1511
1512 if (type & UM_BUFCTL) {
1513 mdb_warn("impossible situation: small-slab UM_BUFCTL walk for "
1514 "cache %p\n", addr);
1515 return (WALK_ERR);
1516 }
1517
1518 /*
1519 * Report allocated buffers, skipping buffers in the magazine layer.
1520 * We only get this far for small-slab caches.
1521 */
1522 for (i = 0; ret == WALK_NEXT && i < chunks; i++) {
1523 buf = (char *)kbase + i * chunksize;
1524
1525 if (!valid[i])
1526 continue; /* on slab freelist */
1527
1528 if (magcnt > 0 &&
1529 bsearch(&buf, maglist, magcnt, sizeof (void *),
1530 addrcmp) != NULL)
1531 continue; /* in magazine layer */
1532
1533 ret = umem_walk_callback(wsp, (uintptr_t)buf);
1534 }
1535 return (ret);
1536 }
1537
1538 void
1539 umem_walk_fini(mdb_walk_state_t *wsp)
1540 {
1541 umem_walk_t *umw = wsp->walk_data;
1542 uintptr_t chunksize;
1543 uintptr_t slabsize;
1544
1545 if (umw == NULL)
1546 return;
1547
1548 if (umw->umw_maglist != NULL)
1549 mdb_free(umw->umw_maglist, umw->umw_max * sizeof (void *));
1550
1551 chunksize = umw->umw_cp->cache_chunksize;
1552 slabsize = umw->umw_cp->cache_slabsize;
1553
1554 if (umw->umw_valid != NULL)
1555 mdb_free(umw->umw_valid, slabsize / chunksize);
1556 if (umw->umw_ubase != NULL)
1557 mdb_free(umw->umw_ubase, slabsize + sizeof (umem_bufctl_t));
1558
1559 mdb_free(umw->umw_cp, umw->umw_csize);
1560 mdb_free(umw, sizeof (umem_walk_t));
1561 }
1562
1563 /*ARGSUSED*/
1564 static int
1565 umem_walk_all(uintptr_t addr, const umem_cache_t *c, mdb_walk_state_t *wsp)
1566 {
1567 /*
1568 * Buffers allocated from NOTOUCH caches can also show up as freed
1569 * memory in other caches. This can be a little confusing, so we
1570 * don't walk NOTOUCH caches when walking all caches (thereby assuring
1571 * that "::walk umem" and "::walk freemem" yield disjoint output).
1572 */
1573 if (c->cache_cflags & UMC_NOTOUCH)
1574 return (WALK_NEXT);
1575
1576 if (mdb_pwalk(wsp->walk_data, wsp->walk_callback,
1577 wsp->walk_cbdata, addr) == -1)
1578 return (WALK_DONE);
1579
1580 return (WALK_NEXT);
1581 }
1582
1583 #define UMEM_WALK_ALL(name, wsp) { \
1584 wsp->walk_data = (name); \
1585 if (mdb_walk("umem_cache", (mdb_walk_cb_t)umem_walk_all, wsp) == -1) \
1586 return (WALK_ERR); \
1587 return (WALK_DONE); \
1588 }
1589
1590 int
1591 umem_walk_init(mdb_walk_state_t *wsp)
1592 {
1593 if (wsp->walk_arg != NULL)
1594 wsp->walk_addr = (uintptr_t)wsp->walk_arg;
1595
1596 if (wsp->walk_addr == NULL)
1597 UMEM_WALK_ALL("umem", wsp);
1598 return (umem_walk_init_common(wsp, UM_ALLOCATED));
1599 }
1600
1601 int
1602 bufctl_walk_init(mdb_walk_state_t *wsp)
1603 {
1604 if (wsp->walk_addr == NULL)
1605 UMEM_WALK_ALL("bufctl", wsp);
1606 return (umem_walk_init_common(wsp, UM_ALLOCATED | UM_BUFCTL));
1607 }
1608
1609 int
1610 freemem_walk_init(mdb_walk_state_t *wsp)
1611 {
1612 if (wsp->walk_addr == NULL)
1613 UMEM_WALK_ALL("freemem", wsp);
1614 return (umem_walk_init_common(wsp, UM_FREE));
1615 }
1616
1617 int
1618 freectl_walk_init(mdb_walk_state_t *wsp)
1619 {
1620 if (wsp->walk_addr == NULL)
1621 UMEM_WALK_ALL("freectl", wsp);
1622 return (umem_walk_init_common(wsp, UM_FREE | UM_BUFCTL));
1623 }
1624
1625 typedef struct bufctl_history_walk {
1626 void *bhw_next;
1627 umem_cache_t *bhw_cache;
1628 umem_slab_t *bhw_slab;
1629 hrtime_t bhw_timestamp;
1630 } bufctl_history_walk_t;
1631
1632 int
1633 bufctl_history_walk_init(mdb_walk_state_t *wsp)
1634 {
1635 bufctl_history_walk_t *bhw;
1636 umem_bufctl_audit_t bc;
1637 umem_bufctl_audit_t bcn;
1638
1639 if (wsp->walk_addr == NULL) {
1640 mdb_warn("bufctl_history walk doesn't support global walks\n");
1641 return (WALK_ERR);
1642 }
1643
1644 if (mdb_vread(&bc, sizeof (bc), wsp->walk_addr) == -1) {
1645 mdb_warn("unable to read bufctl at %p", wsp->walk_addr);
1646 return (WALK_ERR);
1647 }
1648
1649 bhw = mdb_zalloc(sizeof (*bhw), UM_SLEEP);
1650 bhw->bhw_timestamp = 0;
1651 bhw->bhw_cache = bc.bc_cache;
1652 bhw->bhw_slab = bc.bc_slab;
1653
1654 /*
1655 * sometimes the first log entry matches the base bufctl; in that
1656 * case, skip the base bufctl.
1657 */
1658 if (bc.bc_lastlog != NULL &&
1659 mdb_vread(&bcn, sizeof (bcn), (uintptr_t)bc.bc_lastlog) != -1 &&
1660 bc.bc_addr == bcn.bc_addr &&
1661 bc.bc_cache == bcn.bc_cache &&
1662 bc.bc_slab == bcn.bc_slab &&
1663 bc.bc_timestamp == bcn.bc_timestamp &&
1664 bc.bc_thread == bcn.bc_thread)
1665 bhw->bhw_next = bc.bc_lastlog;
1666 else
1667 bhw->bhw_next = (void *)wsp->walk_addr;
1668
1669 wsp->walk_addr = (uintptr_t)bc.bc_addr;
1670 wsp->walk_data = bhw;
1671
1672 return (WALK_NEXT);
1673 }
1674
1675 int
1676 bufctl_history_walk_step(mdb_walk_state_t *wsp)
1677 {
1678 bufctl_history_walk_t *bhw = wsp->walk_data;
1679 uintptr_t addr = (uintptr_t)bhw->bhw_next;
1680 uintptr_t baseaddr = wsp->walk_addr;
1681 umem_bufctl_audit_t *b;
1682 UMEM_LOCAL_BUFCTL_AUDIT(&b);
1683
1684 if (addr == NULL)
1685 return (WALK_DONE);
1686
1687 if (mdb_vread(b, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
1688 mdb_warn("unable to read bufctl at %p", bhw->bhw_next);
1689 return (WALK_ERR);
1690 }
1691
1692 /*
1693 * The bufctl is only valid if the address, cache, and slab are
1694 * correct. We also check that the timestamp is decreasing, to
1695 * prevent infinite loops.
1696 */
1697 if ((uintptr_t)b->bc_addr != baseaddr ||
1698 b->bc_cache != bhw->bhw_cache ||
1699 b->bc_slab != bhw->bhw_slab ||
1700 (bhw->bhw_timestamp != 0 && b->bc_timestamp >= bhw->bhw_timestamp))
1701 return (WALK_DONE);
1702
1703 bhw->bhw_next = b->bc_lastlog;
1704 bhw->bhw_timestamp = b->bc_timestamp;
1705
1706 return (wsp->walk_callback(addr, b, wsp->walk_cbdata));
1707 }
1708
1709 void
1710 bufctl_history_walk_fini(mdb_walk_state_t *wsp)
1711 {
1712 bufctl_history_walk_t *bhw = wsp->walk_data;
1713
1714 mdb_free(bhw, sizeof (*bhw));
1715 }
1716
1717 typedef struct umem_log_walk {
1718 umem_bufctl_audit_t *ulw_base;
1719 umem_bufctl_audit_t **ulw_sorted;
1720 umem_log_header_t ulw_lh;
1721 size_t ulw_size;
1722 size_t ulw_maxndx;
1723 size_t ulw_ndx;
1724 } umem_log_walk_t;
1725
1726 int
1727 umem_log_walk_init(mdb_walk_state_t *wsp)
1728 {
1729 uintptr_t lp = wsp->walk_addr;
1730 umem_log_walk_t *ulw;
1731 umem_log_header_t *lhp;
1732 int maxndx, i, j, k;
1733
1734 /*
1735 * By default (global walk), walk the umem_transaction_log. Otherwise
1736 * read the log whose umem_log_header_t is stored at walk_addr.
1737 */
1738 if (lp == NULL && umem_readvar(&lp, "umem_transaction_log") == -1) {
1739 mdb_warn("failed to read 'umem_transaction_log'");
1740 return (WALK_ERR);
1741 }
1742
1743 if (lp == NULL) {
1744 mdb_warn("log is disabled\n");
1745 return (WALK_ERR);
1746 }
1747
1748 ulw = mdb_zalloc(sizeof (umem_log_walk_t), UM_SLEEP);
1749 lhp = &ulw->ulw_lh;
1750
1751 if (mdb_vread(lhp, sizeof (umem_log_header_t), lp) == -1) {
1752 mdb_warn("failed to read log header at %p", lp);
1753 mdb_free(ulw, sizeof (umem_log_walk_t));
1754 return (WALK_ERR);
1755 }
1756
1757 ulw->ulw_size = lhp->lh_chunksize * lhp->lh_nchunks;
1758 ulw->ulw_base = mdb_alloc(ulw->ulw_size, UM_SLEEP);
1759 maxndx = lhp->lh_chunksize / UMEM_BUFCTL_AUDIT_SIZE - 1;
1760
1761 if (mdb_vread(ulw->ulw_base, ulw->ulw_size,
1762 (uintptr_t)lhp->lh_base) == -1) {
1763 mdb_warn("failed to read log at base %p", lhp->lh_base);
1764 mdb_free(ulw->ulw_base, ulw->ulw_size);
1765 mdb_free(ulw, sizeof (umem_log_walk_t));
1766 return (WALK_ERR);
1767 }
1768
1769 ulw->ulw_sorted = mdb_alloc(maxndx * lhp->lh_nchunks *
1770 sizeof (umem_bufctl_audit_t *), UM_SLEEP);
1771
1772 for (i = 0, k = 0; i < lhp->lh_nchunks; i++) {
1773 caddr_t chunk = (caddr_t)
1774 ((uintptr_t)ulw->ulw_base + i * lhp->lh_chunksize);
1775
1776 for (j = 0; j < maxndx; j++) {
1777 /* LINTED align */
1778 ulw->ulw_sorted[k++] = (umem_bufctl_audit_t *)chunk;
1779 chunk += UMEM_BUFCTL_AUDIT_SIZE;
1780 }
1781 }
1782
1783 qsort(ulw->ulw_sorted, k, sizeof (umem_bufctl_audit_t *),
1784 (int(*)(const void *, const void *))bufctlcmp);
1785
1786 ulw->ulw_maxndx = k;
1787 wsp->walk_data = ulw;
1788
1789 return (WALK_NEXT);
1790 }
1791
1792 int
1793 umem_log_walk_step(mdb_walk_state_t *wsp)
1794 {
1795 umem_log_walk_t *ulw = wsp->walk_data;
1796 umem_bufctl_audit_t *bcp;
1797
1798 if (ulw->ulw_ndx == ulw->ulw_maxndx)
1799 return (WALK_DONE);
1800
1801 bcp = ulw->ulw_sorted[ulw->ulw_ndx++];
1802
1803 return (wsp->walk_callback((uintptr_t)bcp - (uintptr_t)ulw->ulw_base +
1804 (uintptr_t)ulw->ulw_lh.lh_base, bcp, wsp->walk_cbdata));
1805 }
1806
1807 void
1808 umem_log_walk_fini(mdb_walk_state_t *wsp)
1809 {
1810 umem_log_walk_t *ulw = wsp->walk_data;
1811
1812 mdb_free(ulw->ulw_base, ulw->ulw_size);
1813 mdb_free(ulw->ulw_sorted, ulw->ulw_maxndx *
1814 sizeof (umem_bufctl_audit_t *));
1815 mdb_free(ulw, sizeof (umem_log_walk_t));
1816 }
1817
1818 typedef struct allocdby_bufctl {
1819 uintptr_t abb_addr;
1820 hrtime_t abb_ts;
1821 } allocdby_bufctl_t;
1822
1823 typedef struct allocdby_walk {
1824 const char *abw_walk;
1825 uintptr_t abw_thread;
1826 size_t abw_nbufs;
1827 size_t abw_size;
1828 allocdby_bufctl_t *abw_buf;
1829 size_t abw_ndx;
1830 } allocdby_walk_t;
1831
1832 int
1833 allocdby_walk_bufctl(uintptr_t addr, const umem_bufctl_audit_t *bcp,
1834 allocdby_walk_t *abw)
1835 {
1836 if ((uintptr_t)bcp->bc_thread != abw->abw_thread)
1837 return (WALK_NEXT);
1838
1839 if (abw->abw_nbufs == abw->abw_size) {
1840 allocdby_bufctl_t *buf;
1841 size_t oldsize = sizeof (allocdby_bufctl_t) * abw->abw_size;
1842
1843 buf = mdb_zalloc(oldsize << 1, UM_SLEEP);
1844
1845 bcopy(abw->abw_buf, buf, oldsize);
1846 mdb_free(abw->abw_buf, oldsize);
1847
1848 abw->abw_size <<= 1;
1849 abw->abw_buf = buf;
1850 }
1851
1852 abw->abw_buf[abw->abw_nbufs].abb_addr = addr;
1853 abw->abw_buf[abw->abw_nbufs].abb_ts = bcp->bc_timestamp;
1854 abw->abw_nbufs++;
1855
1856 return (WALK_NEXT);
1857 }
1858
1859 /*ARGSUSED*/
1860 int
1861 allocdby_walk_cache(uintptr_t addr, const umem_cache_t *c, allocdby_walk_t *abw)
1862 {
1863 if (mdb_pwalk(abw->abw_walk, (mdb_walk_cb_t)allocdby_walk_bufctl,
1864 abw, addr) == -1) {
1865 mdb_warn("couldn't walk bufctl for cache %p", addr);
1866 return (WALK_DONE);
1867 }
1868
1869 return (WALK_NEXT);
1870 }
1871
1872 static int
1873 allocdby_cmp(const allocdby_bufctl_t *lhs, const allocdby_bufctl_t *rhs)
1874 {
1875 if (lhs->abb_ts < rhs->abb_ts)
1876 return (1);
1877 if (lhs->abb_ts > rhs->abb_ts)
1878 return (-1);
1879 return (0);
1880 }
1881
1882 static int
1883 allocdby_walk_init_common(mdb_walk_state_t *wsp, const char *walk)
1884 {
1885 allocdby_walk_t *abw;
1886
1887 if (wsp->walk_addr == NULL) {
1888 mdb_warn("allocdby walk doesn't support global walks\n");
1889 return (WALK_ERR);
1890 }
1891
1892 abw = mdb_zalloc(sizeof (allocdby_walk_t), UM_SLEEP);
1893
1894 abw->abw_thread = wsp->walk_addr;
1895 abw->abw_walk = walk;
1896 abw->abw_size = 128; /* something reasonable */
1897 abw->abw_buf =
1898 mdb_zalloc(abw->abw_size * sizeof (allocdby_bufctl_t), UM_SLEEP);
1899
1900 wsp->walk_data = abw;
1901
1902 if (mdb_walk("umem_cache",
1903 (mdb_walk_cb_t)allocdby_walk_cache, abw) == -1) {
1904 mdb_warn("couldn't walk umem_cache");
1905 allocdby_walk_fini(wsp);
1906 return (WALK_ERR);
1907 }
1908
1909 qsort(abw->abw_buf, abw->abw_nbufs, sizeof (allocdby_bufctl_t),
1910 (int(*)(const void *, const void *))allocdby_cmp);
1911
1912 return (WALK_NEXT);
1913 }
1914
1915 int
1916 allocdby_walk_init(mdb_walk_state_t *wsp)
1917 {
1918 return (allocdby_walk_init_common(wsp, "bufctl"));
1919 }
1920
1921 int
1922 freedby_walk_init(mdb_walk_state_t *wsp)
1923 {
1924 return (allocdby_walk_init_common(wsp, "freectl"));
1925 }
1926
1927 int
1928 allocdby_walk_step(mdb_walk_state_t *wsp)
1929 {
1930 allocdby_walk_t *abw = wsp->walk_data;
1931 uintptr_t addr;
1932 umem_bufctl_audit_t *bcp;
1933 UMEM_LOCAL_BUFCTL_AUDIT(&bcp);
1934
1935 if (abw->abw_ndx == abw->abw_nbufs)
1936 return (WALK_DONE);
1937
1938 addr = abw->abw_buf[abw->abw_ndx++].abb_addr;
1939
1940 if (mdb_vread(bcp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
1941 mdb_warn("couldn't read bufctl at %p", addr);
1942 return (WALK_DONE);
1943 }
1944
1945 return (wsp->walk_callback(addr, bcp, wsp->walk_cbdata));
1946 }
1947
1948 void
1949 allocdby_walk_fini(mdb_walk_state_t *wsp)
1950 {
1951 allocdby_walk_t *abw = wsp->walk_data;
1952
1953 mdb_free(abw->abw_buf, sizeof (allocdby_bufctl_t) * abw->abw_size);
1954 mdb_free(abw, sizeof (allocdby_walk_t));
1955 }
1956
1957 /*ARGSUSED*/
1958 int
1959 allocdby_walk(uintptr_t addr, const umem_bufctl_audit_t *bcp, void *ignored)
1960 {
1961 char c[MDB_SYM_NAMLEN];
1962 GElf_Sym sym;
1963 int i;
1964
1965 mdb_printf("%0?p %12llx ", addr, bcp->bc_timestamp);
1966 for (i = 0; i < bcp->bc_depth; i++) {
1967 if (mdb_lookup_by_addr(bcp->bc_stack[i],
1968 MDB_SYM_FUZZY, c, sizeof (c), &sym) == -1)
1969 continue;
1970 if (is_umem_sym(c, "umem_"))
1971 continue;
1972 mdb_printf("%s+0x%lx",
1973 c, bcp->bc_stack[i] - (uintptr_t)sym.st_value);
1974 break;
1975 }
1976 mdb_printf("\n");
1977
1978 return (WALK_NEXT);
1979 }
1980
1981 static int
1982 allocdby_common(uintptr_t addr, uint_t flags, const char *w)
1983 {
1984 if (!(flags & DCMD_ADDRSPEC))
1985 return (DCMD_USAGE);
1986
1987 mdb_printf("%-?s %12s %s\n", "BUFCTL", "TIMESTAMP", "CALLER");
1988
1989 if (mdb_pwalk(w, (mdb_walk_cb_t)allocdby_walk, NULL, addr) == -1) {
1990 mdb_warn("can't walk '%s' for %p", w, addr);
1991 return (DCMD_ERR);
1992 }
1993
1994 return (DCMD_OK);
1995 }
1996
1997 /*ARGSUSED*/
1998 int
1999 allocdby(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2000 {
2001 return (allocdby_common(addr, flags, "allocdby"));
2002 }
2003
2004 /*ARGSUSED*/
2005 int
2006 freedby(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2007 {
2008 return (allocdby_common(addr, flags, "freedby"));
2009 }
2010
2011 typedef struct whatis_info {
2012 mdb_whatis_t *wi_w;
2013 const umem_cache_t *wi_cache;
2014 const vmem_t *wi_vmem;
2015 vmem_t *wi_msb_arena;
2016 size_t wi_slab_size;
2017 int wi_slab_found;
2018 uint_t wi_freemem;
2019 } whatis_info_t;
2020
2021 /* call one of our dcmd functions with "-v" and the provided address */
2022 static void
2023 whatis_call_printer(mdb_dcmd_f *dcmd, uintptr_t addr)
2024 {
2025 mdb_arg_t a;
2026 a.a_type = MDB_TYPE_STRING;
2027 a.a_un.a_str = "-v";
2028
2029 mdb_printf(":\n");
2030 (void) (*dcmd)(addr, DCMD_ADDRSPEC, 1, &a);
2031 }
2032
2033 static void
2034 whatis_print_umem(whatis_info_t *wi, uintptr_t maddr, uintptr_t addr,
2035 uintptr_t baddr)
2036 {
2037 mdb_whatis_t *w = wi->wi_w;
2038 const umem_cache_t *cp = wi->wi_cache;
2039 int quiet = (mdb_whatis_flags(w) & WHATIS_QUIET);
2040
2041 int call_printer = (!quiet && (cp->cache_flags & UMF_AUDIT));
2042
2043 mdb_whatis_report_object(w, maddr, addr, "");
2044
2045 if (baddr != 0 && !call_printer)
2046 mdb_printf("bufctl %p ", baddr);
2047
2048 mdb_printf("%s from %s",
2049 (wi->wi_freemem == FALSE) ? "allocated" : "freed", cp->cache_name);
2050
2051 if (call_printer && baddr != 0) {
2052 whatis_call_printer(bufctl, baddr);
2053 return;
2054 }
2055 mdb_printf("\n");
2056 }
2057
2058 /*ARGSUSED*/
2059 static int
2060 whatis_walk_umem(uintptr_t addr, void *ignored, whatis_info_t *wi)
2061 {
2062 mdb_whatis_t *w = wi->wi_w;
2063
2064 uintptr_t cur;
2065 size_t size = wi->wi_cache->cache_bufsize;
2066
2067 while (mdb_whatis_match(w, addr, size, &cur))
2068 whatis_print_umem(wi, cur, addr, NULL);
2069
2070 return (WHATIS_WALKRET(w));
2071 }
2072
2073 /*ARGSUSED*/
2074 static int
2075 whatis_walk_bufctl(uintptr_t baddr, const umem_bufctl_t *bcp, whatis_info_t *wi)
2076 {
2077 mdb_whatis_t *w = wi->wi_w;
2078
2079 uintptr_t cur;
2080 uintptr_t addr = (uintptr_t)bcp->bc_addr;
2081 size_t size = wi->wi_cache->cache_bufsize;
2082
2083 while (mdb_whatis_match(w, addr, size, &cur))
2084 whatis_print_umem(wi, cur, addr, baddr);
2085
2086 return (WHATIS_WALKRET(w));
2087 }
2088
2089
2090 static int
2091 whatis_walk_seg(uintptr_t addr, const vmem_seg_t *vs, whatis_info_t *wi)
2092 {
2093 mdb_whatis_t *w = wi->wi_w;
2094
2095 size_t size = vs->vs_end - vs->vs_start;
2096 uintptr_t cur;
2097
2098 /* We're not interested in anything but alloc and free segments */
2099 if (vs->vs_type != VMEM_ALLOC && vs->vs_type != VMEM_FREE)
2100 return (WALK_NEXT);
2101
2102 while (mdb_whatis_match(w, vs->vs_start, size, &cur)) {
2103 mdb_whatis_report_object(w, cur, vs->vs_start, "");
2104
2105 /*
2106 * If we're not printing it seperately, provide the vmem_seg
2107 * pointer if it has a stack trace.
2108 */
2109 if ((mdb_whatis_flags(w) & WHATIS_QUIET) &&
2110 ((mdb_whatis_flags(w) & WHATIS_BUFCTL) != 0 ||
2111 (vs->vs_type == VMEM_ALLOC && vs->vs_depth != 0))) {
2112 mdb_printf("vmem_seg %p ", addr);
2113 }
2114
2115 mdb_printf("%s from %s vmem arena",
2116 (vs->vs_type == VMEM_ALLOC) ? "allocated" : "freed",
2117 wi->wi_vmem->vm_name);
2118
2119 if (!(mdb_whatis_flags(w) & WHATIS_QUIET))
2120 whatis_call_printer(vmem_seg, addr);
2121 else
2122 mdb_printf("\n");
2123 }
2124
2125 return (WHATIS_WALKRET(w));
2126 }
2127
2128 static int
2129 whatis_walk_vmem(uintptr_t addr, const vmem_t *vmem, whatis_info_t *wi)
2130 {
2131 mdb_whatis_t *w = wi->wi_w;
2132 const char *nm = vmem->vm_name;
2133 wi->wi_vmem = vmem;
2134
2135 if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
2136 mdb_printf("Searching vmem arena %s...\n", nm);
2137
2138 if (mdb_pwalk("vmem_seg",
2139 (mdb_walk_cb_t)whatis_walk_seg, wi, addr) == -1) {
2140 mdb_warn("can't walk vmem seg for %p", addr);
2141 return (WALK_NEXT);
2142 }
2143
2144 return (WHATIS_WALKRET(w));
2145 }
2146
2147 /*ARGSUSED*/
2148 static int
2149 whatis_walk_slab(uintptr_t saddr, const umem_slab_t *sp, whatis_info_t *wi)
2150 {
2151 mdb_whatis_t *w = wi->wi_w;
2152
2153 /* It must overlap with the slab data, or it's not interesting */
2154 if (mdb_whatis_overlaps(w,
2155 (uintptr_t)sp->slab_base, wi->wi_slab_size)) {
2156 wi->wi_slab_found++;
2157 return (WALK_DONE);
2158 }
2159 return (WALK_NEXT);
2160 }
2161
2162 static int
2163 whatis_walk_cache(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
2164 {
2165 mdb_whatis_t *w = wi->wi_w;
2166 char *walk, *freewalk;
2167 mdb_walk_cb_t func;
2168 int do_bufctl;
2169
2170 /* Override the '-b' flag as necessary */
2171 if (!(c->cache_flags & UMF_HASH))
2172 do_bufctl = FALSE; /* no bufctls to walk */
2173 else if (c->cache_flags & UMF_AUDIT)
2174 do_bufctl = TRUE; /* we always want debugging info */
2175 else
2176 do_bufctl = ((mdb_whatis_flags(w) & WHATIS_BUFCTL) != 0);
2177
2178 if (do_bufctl) {
2179 walk = "bufctl";
2180 freewalk = "freectl";
2181 func = (mdb_walk_cb_t)whatis_walk_bufctl;
2182 } else {
2183 walk = "umem";
2184 freewalk = "freemem";
2185 func = (mdb_walk_cb_t)whatis_walk_umem;
2186 }
2187
2188 wi->wi_cache = c;
2189
2190 if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
2191 mdb_printf("Searching %s...\n", c->cache_name);
2192
2193 /*
2194 * If more then two buffers live on each slab, figure out if we're
2195 * interested in anything in any slab before doing the more expensive
2196 * umem/freemem (bufctl/freectl) walkers.
2197 */
2198 wi->wi_slab_size = c->cache_slabsize - c->cache_maxcolor;
2199 if (!(c->cache_flags & UMF_HASH))
2200 wi->wi_slab_size -= sizeof (umem_slab_t);
2201
2202 if ((wi->wi_slab_size / c->cache_chunksize) > 2) {
2203 wi->wi_slab_found = 0;
2204 if (mdb_pwalk("umem_slab", (mdb_walk_cb_t)whatis_walk_slab, wi,
2205 addr) == -1) {
2206 mdb_warn("can't find umem_slab walker");
2207 return (WALK_DONE);
2208 }
2209 if (wi->wi_slab_found == 0)
2210 return (WALK_NEXT);
2211 }
2212
2213 wi->wi_freemem = FALSE;
2214 if (mdb_pwalk(walk, func, wi, addr) == -1) {
2215 mdb_warn("can't find %s walker", walk);
2216 return (WALK_DONE);
2217 }
2218
2219 if (mdb_whatis_done(w))
2220 return (WALK_DONE);
2221
2222 /*
2223 * We have searched for allocated memory; now search for freed memory.
2224 */
2225 if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
2226 mdb_printf("Searching %s for free memory...\n", c->cache_name);
2227
2228 wi->wi_freemem = TRUE;
2229
2230 if (mdb_pwalk(freewalk, func, wi, addr) == -1) {
2231 mdb_warn("can't find %s walker", freewalk);
2232 return (WALK_DONE);
2233 }
2234
2235 return (WHATIS_WALKRET(w));
2236 }
2237
2238 static int
2239 whatis_walk_touch(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
2240 {
2241 if (c->cache_arena == wi->wi_msb_arena ||
2242 (c->cache_cflags & UMC_NOTOUCH))
2243 return (WALK_NEXT);
2244
2245 return (whatis_walk_cache(addr, c, wi));
2246 }
2247
2248 static int
2249 whatis_walk_metadata(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
2250 {
2251 if (c->cache_arena != wi->wi_msb_arena)
2252 return (WALK_NEXT);
2253
2254 return (whatis_walk_cache(addr, c, wi));
2255 }
2256
2257 static int
2258 whatis_walk_notouch(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
2259 {
2260 if (c->cache_arena == wi->wi_msb_arena ||
2261 !(c->cache_cflags & UMC_NOTOUCH))
2262 return (WALK_NEXT);
2263
2264 return (whatis_walk_cache(addr, c, wi));
2265 }
2266
2267 /*ARGSUSED*/
2268 static int
2269 whatis_run_umem(mdb_whatis_t *w, void *ignored)
2270 {
2271 whatis_info_t wi;
2272
2273 bzero(&wi, sizeof (wi));
2274 wi.wi_w = w;
2275
2276 /* umem's metadata is allocated from the umem_internal_arena */
2277 if (umem_readvar(&wi.wi_msb_arena, "umem_internal_arena") == -1)
2278 mdb_warn("unable to readvar \"umem_internal_arena\"");
2279
2280 /*
2281 * We process umem caches in the following order:
2282 *
2283 * non-UMC_NOTOUCH, non-metadata (typically the most interesting)
2284 * metadata (can be huge with UMF_AUDIT)
2285 * UMC_NOTOUCH, non-metadata (see umem_walk_all())
2286 */
2287 if (mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_touch,
2288 &wi) == -1 ||
2289 mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_metadata,
2290 &wi) == -1 ||
2291 mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_notouch,
2292 &wi) == -1) {
2293 mdb_warn("couldn't find umem_cache walker");
2294 return (1);
2295 }
2296 return (0);
2297 }
2298
2299 /*ARGSUSED*/
2300 static int
2301 whatis_run_vmem(mdb_whatis_t *w, void *ignored)
2302 {
2303 whatis_info_t wi;
2304
2305 bzero(&wi, sizeof (wi));
2306 wi.wi_w = w;
2307
2308 if (mdb_walk("vmem_postfix",
2309 (mdb_walk_cb_t)whatis_walk_vmem, &wi) == -1) {
2310 mdb_warn("couldn't find vmem_postfix walker");
2311 return (1);
2312 }
2313 return (0);
2314 }
2315
2316 int
2317 umem_init(void)
2318 {
2319 mdb_walker_t w = {
2320 "umem_cache", "walk list of umem caches", umem_cache_walk_init,
2321 umem_cache_walk_step, umem_cache_walk_fini
2322 };
2323
2324 if (mdb_add_walker(&w) == -1) {
2325 mdb_warn("failed to add umem_cache walker");
2326 return (-1);
2327 }
2328
2329 if (umem_update_variables() == -1)
2330 return (-1);
2331
2332 /* install a callback so that our variables are always up-to-date */
2333 (void) mdb_callback_add(MDB_CALLBACK_STCHG, umem_statechange_cb, NULL);
2334 umem_statechange_cb(NULL);
2335
2336 /*
2337 * Register our ::whatis callbacks.
2338 */
2339 mdb_whatis_register("umem", whatis_run_umem, NULL,
2340 WHATIS_PRIO_ALLOCATOR, WHATIS_REG_NO_ID);
2341 mdb_whatis_register("vmem", whatis_run_vmem, NULL,
2342 WHATIS_PRIO_ALLOCATOR, WHATIS_REG_NO_ID);
2343
2344 return (0);
2345 }
2346
2347 typedef struct umem_log_cpu {
2348 uintptr_t umc_low;
2349 uintptr_t umc_high;
2350 } umem_log_cpu_t;
2351
2352 int
2353 umem_log_walk(uintptr_t addr, const umem_bufctl_audit_t *b, umem_log_cpu_t *umc)
2354 {
2355 int i;
2356
2357 for (i = 0; i < umem_max_ncpus; i++) {
2358 if (addr >= umc[i].umc_low && addr < umc[i].umc_high)
2359 break;
2360 }
2361
2362 if (i == umem_max_ncpus)
2363 mdb_printf(" ");
2364 else
2365 mdb_printf("%3d", i);
2366
2367 mdb_printf(" %0?p %0?p %16llx %0?p\n", addr, b->bc_addr,
2368 b->bc_timestamp, b->bc_thread);
2369
2370 return (WALK_NEXT);
2371 }
2372
2373 /*ARGSUSED*/
2374 int
2375 umem_log(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2376 {
2377 umem_log_header_t lh;
2378 umem_cpu_log_header_t clh;
2379 uintptr_t lhp, clhp;
2380 umem_log_cpu_t *umc;
2381 int i;
2382
2383 if (umem_readvar(&lhp, "umem_transaction_log") == -1) {
2384 mdb_warn("failed to read 'umem_transaction_log'");
2385 return (DCMD_ERR);
2386 }
2387
2388 if (lhp == NULL) {
2389 mdb_warn("no umem transaction log\n");
2390 return (DCMD_ERR);
2391 }
2392
2393 if (mdb_vread(&lh, sizeof (umem_log_header_t), lhp) == -1) {
2394 mdb_warn("failed to read log header at %p", lhp);
2395 return (DCMD_ERR);
2396 }
2397
2398 clhp = lhp + ((uintptr_t)&lh.lh_cpu[0] - (uintptr_t)&lh);
2399
2400 umc = mdb_zalloc(sizeof (umem_log_cpu_t) * umem_max_ncpus,
2401 UM_SLEEP | UM_GC);
2402
2403 for (i = 0; i < umem_max_ncpus; i++) {
2404 if (mdb_vread(&clh, sizeof (clh), clhp) == -1) {
2405 mdb_warn("cannot read cpu %d's log header at %p",
2406 i, clhp);
2407 return (DCMD_ERR);
2408 }
2409
2410 umc[i].umc_low = clh.clh_chunk * lh.lh_chunksize +
2411 (uintptr_t)lh.lh_base;
2412 umc[i].umc_high = (uintptr_t)clh.clh_current;
2413
2414 clhp += sizeof (umem_cpu_log_header_t);
2415 }
2416
2417 if (DCMD_HDRSPEC(flags)) {
2418 mdb_printf("%3s %-?s %-?s %16s %-?s\n", "CPU", "ADDR",
2419 "BUFADDR", "TIMESTAMP", "THREAD");
2420 }
2421
2422 /*
2423 * If we have been passed an address, we'll just print out that
2424 * log entry.
2425 */
2426 if (flags & DCMD_ADDRSPEC) {
2427 umem_bufctl_audit_t *bp;
2428 UMEM_LOCAL_BUFCTL_AUDIT(&bp);
2429
2430 if (mdb_vread(bp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
2431 mdb_warn("failed to read bufctl at %p", addr);
2432 return (DCMD_ERR);
2433 }
2434
2435 (void) umem_log_walk(addr, bp, umc);
2436
2437 return (DCMD_OK);
2438 }
2439
2440 if (mdb_walk("umem_log", (mdb_walk_cb_t)umem_log_walk, umc) == -1) {
2441 mdb_warn("can't find umem log walker");
2442 return (DCMD_ERR);
2443 }
2444
2445 return (DCMD_OK);
2446 }
2447
2448 typedef struct bufctl_history_cb {
2449 int bhc_flags;
2450 int bhc_argc;
2451 const mdb_arg_t *bhc_argv;
2452 int bhc_ret;
2453 } bufctl_history_cb_t;
2454
2455 /*ARGSUSED*/
2456 static int
2457 bufctl_history_callback(uintptr_t addr, const void *ign, void *arg)
2458 {
2459 bufctl_history_cb_t *bhc = arg;
2460
2461 bhc->bhc_ret =
2462 bufctl(addr, bhc->bhc_flags, bhc->bhc_argc, bhc->bhc_argv);
2463
2464 bhc->bhc_flags &= ~DCMD_LOOPFIRST;
2465
2466 return ((bhc->bhc_ret == DCMD_OK)? WALK_NEXT : WALK_DONE);
2467 }
2468
2469 void
2470 bufctl_help(void)
2471 {
2472 mdb_printf("%s\n",
2473 "Display the contents of umem_bufctl_audit_ts, with optional filtering.\n");
2474 mdb_dec_indent(2);
2475 mdb_printf("%<b>OPTIONS%</b>\n");
2476 mdb_inc_indent(2);
2477 mdb_printf("%s",
2478 " -v Display the full content of the bufctl, including its stack trace\n"
2479 " -h retrieve the bufctl's transaction history, if available\n"
2480 " -a addr\n"
2481 " filter out bufctls not involving the buffer at addr\n"
2482 " -c caller\n"
2483 " filter out bufctls without the function/PC in their stack trace\n"
2484 " -e earliest\n"
2485 " filter out bufctls timestamped before earliest\n"
2486 " -l latest\n"
2487 " filter out bufctls timestamped after latest\n"
2488 " -t thread\n"
2489 " filter out bufctls not involving thread\n");
2490 }
2491
2492 int
2493 bufctl(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2494 {
2495 uint_t verbose = FALSE;
2496 uint_t history = FALSE;
2497 uint_t in_history = FALSE;
2498 uintptr_t caller = NULL, thread = NULL;
2499 uintptr_t laddr, haddr, baddr = NULL;
2500 hrtime_t earliest = 0, latest = 0;
2501 int i, depth;
2502 char c[MDB_SYM_NAMLEN];
2503 GElf_Sym sym;
2504 umem_bufctl_audit_t *bcp;
2505 UMEM_LOCAL_BUFCTL_AUDIT(&bcp);
2506
2507 if (mdb_getopts(argc, argv,
2508 'v', MDB_OPT_SETBITS, TRUE, &verbose,
2509 'h', MDB_OPT_SETBITS, TRUE, &history,
2510 'H', MDB_OPT_SETBITS, TRUE, &in_history, /* internal */
2511 'c', MDB_OPT_UINTPTR, &caller,
2512 't', MDB_OPT_UINTPTR, &thread,
2513 'e', MDB_OPT_UINT64, &earliest,
2514 'l', MDB_OPT_UINT64, &latest,
2515 'a', MDB_OPT_UINTPTR, &baddr, NULL) != argc)
2516 return (DCMD_USAGE);
2517
2518 if (!(flags & DCMD_ADDRSPEC))
2519 return (DCMD_USAGE);
2520
2521 if (in_history && !history)
2522 return (DCMD_USAGE);
2523
2524 if (history && !in_history) {
2525 mdb_arg_t *nargv = mdb_zalloc(sizeof (*nargv) * (argc + 1),
2526 UM_SLEEP | UM_GC);
2527 bufctl_history_cb_t bhc;
2528
2529 nargv[0].a_type = MDB_TYPE_STRING;
2530 nargv[0].a_un.a_str = "-H"; /* prevent recursion */
2531
2532 for (i = 0; i < argc; i++)
2533 nargv[i + 1] = argv[i];
2534
2535 /*
2536 * When in history mode, we treat each element as if it
2537 * were in a seperate loop, so that the headers group
2538 * bufctls with similar histories.
2539 */
2540 bhc.bhc_flags = flags | DCMD_LOOP | DCMD_LOOPFIRST;
2541 bhc.bhc_argc = argc + 1;
2542 bhc.bhc_argv = nargv;
2543 bhc.bhc_ret = DCMD_OK;
2544
2545 if (mdb_pwalk("bufctl_history", bufctl_history_callback, &bhc,
2546 addr) == -1) {
2547 mdb_warn("unable to walk bufctl_history");
2548 return (DCMD_ERR);
2549 }
2550
2551 if (bhc.bhc_ret == DCMD_OK && !(flags & DCMD_PIPE_OUT))
2552 mdb_printf("\n");
2553
2554 return (bhc.bhc_ret);
2555 }
2556
2557 if (DCMD_HDRSPEC(flags) && !(flags & DCMD_PIPE_OUT)) {
2558 if (verbose) {
2559 mdb_printf("%16s %16s %16s %16s\n"
2560 "%<u>%16s %16s %16s %16s%</u>\n",
2561 "ADDR", "BUFADDR", "TIMESTAMP", "THREAD",
2562 "", "CACHE", "LASTLOG", "CONTENTS");
2563 } else {
2564 mdb_printf("%<u>%-?s %-?s %-12s %5s %s%</u>\n",
2565 "ADDR", "BUFADDR", "TIMESTAMP", "THRD", "CALLER");
2566 }
2567 }
2568
2569 if (mdb_vread(bcp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
2570 mdb_warn("couldn't read bufctl at %p", addr);
2571 return (DCMD_ERR);
2572 }
2573
2574 /*
2575 * Guard against bogus bc_depth in case the bufctl is corrupt or
2576 * the address does not really refer to a bufctl.
2577 */
2578 depth = MIN(bcp->bc_depth, umem_stack_depth);
2579
2580 if (caller != NULL) {
2581 laddr = caller;
2582 haddr = caller + sizeof (caller);
2583
2584 if (mdb_lookup_by_addr(caller, MDB_SYM_FUZZY, c, sizeof (c),
2585 &sym) != -1 && caller == (uintptr_t)sym.st_value) {
2586 /*
2587 * We were provided an exact symbol value; any
2588 * address in the function is valid.
2589 */
2590 laddr = (uintptr_t)sym.st_value;
2591 haddr = (uintptr_t)sym.st_value + sym.st_size;
2592 }
2593
2594 for (i = 0; i < depth; i++)
2595 if (bcp->bc_stack[i] >= laddr &&
2596 bcp->bc_stack[i] < haddr)
2597 break;
2598
2599 if (i == depth)
2600 return (DCMD_OK);
2601 }
2602
2603 if (thread != NULL && (uintptr_t)bcp->bc_thread != thread)
2604 return (DCMD_OK);
2605
2606 if (earliest != 0 && bcp->bc_timestamp < earliest)
2607 return (DCMD_OK);
2608
2609 if (latest != 0 && bcp->bc_timestamp > latest)
2610 return (DCMD_OK);
2611
2612 if (baddr != 0 && (uintptr_t)bcp->bc_addr != baddr)
2613 return (DCMD_OK);
2614
2615 if (flags & DCMD_PIPE_OUT) {
2616 mdb_printf("%#r\n", addr);
2617 return (DCMD_OK);
2618 }
2619
2620 if (verbose) {
2621 mdb_printf(
2622 "%<b>%16p%</b> %16p %16llx %16d\n"
2623 "%16s %16p %16p %16p\n",
2624 addr, bcp->bc_addr, bcp->bc_timestamp, bcp->bc_thread,
2625 "", bcp->bc_cache, bcp->bc_lastlog, bcp->bc_contents);
2626
2627 mdb_inc_indent(17);
2628 for (i = 0; i < depth; i++)
2629 mdb_printf("%a\n", bcp->bc_stack[i]);
2630 mdb_dec_indent(17);
2631 mdb_printf("\n");
2632 } else {
2633 mdb_printf("%0?p %0?p %12llx %5d", addr, bcp->bc_addr,
2634 bcp->bc_timestamp, bcp->bc_thread);
2635
2636 for (i = 0; i < depth; i++) {
2637 if (mdb_lookup_by_addr(bcp->bc_stack[i],
2638 MDB_SYM_FUZZY, c, sizeof (c), &sym) == -1)
2639 continue;
2640 if (is_umem_sym(c, "umem_"))
2641 continue;
2642 mdb_printf(" %a\n", bcp->bc_stack[i]);
2643 break;
2644 }
2645
2646 if (i >= depth)
2647 mdb_printf("\n");
2648 }
2649
2650 return (DCMD_OK);
2651 }
2652
2653 /*ARGSUSED*/
2654 int
2655 bufctl_audit(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2656 {
2657 mdb_arg_t a;
2658
2659 if (!(flags & DCMD_ADDRSPEC))
2660 return (DCMD_USAGE);
2661
2662 if (argc != 0)
2663 return (DCMD_USAGE);
2664
2665 a.a_type = MDB_TYPE_STRING;
2666 a.a_un.a_str = "-v";
2667
2668 return (bufctl(addr, flags, 1, &a));
2669 }
2670
2671 typedef struct umem_verify {
2672 uint64_t *umv_buf; /* buffer to read cache contents into */
2673 size_t umv_size; /* number of bytes in umv_buf */
2674 int umv_corruption; /* > 0 if corruption found. */
2675 int umv_besilent; /* report actual corruption sites */
2676 struct umem_cache umv_cache; /* the cache we're operating on */
2677 } umem_verify_t;
2678
2679 /*
2680 * verify_pattern()
2681 * verify that buf is filled with the pattern pat.
2682 */
2683 static int64_t
2684 verify_pattern(uint64_t *buf_arg, size_t size, uint64_t pat)
2685 {
2686 /*LINTED*/
2687 uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
2688 uint64_t *buf;
2689
2690 for (buf = buf_arg; buf < bufend; buf++)
2691 if (*buf != pat)
2692 return ((uintptr_t)buf - (uintptr_t)buf_arg);
2693 return (-1);
2694 }
2695
2696 /*
2697 * verify_buftag()
2698 * verify that btp->bt_bxstat == (bcp ^ pat)
2699 */
2700 static int
2701 verify_buftag(umem_buftag_t *btp, uintptr_t pat)
2702 {
2703 return (btp->bt_bxstat == ((intptr_t)btp->bt_bufctl ^ pat) ? 0 : -1);
2704 }
2705
2706 /*
2707 * verify_free()
2708 * verify the integrity of a free block of memory by checking
2709 * that it is filled with 0xdeadbeef and that its buftag is sane.
2710 */
2711 /*ARGSUSED1*/
2712 static int
2713 verify_free(uintptr_t addr, const void *data, void *private)
2714 {
2715 umem_verify_t *umv = (umem_verify_t *)private;
2716 uint64_t *buf = umv->umv_buf; /* buf to validate */
2717 int64_t corrupt; /* corruption offset */
2718 umem_buftag_t *buftagp; /* ptr to buftag */
2719 umem_cache_t *cp = &umv->umv_cache;
2720 int besilent = umv->umv_besilent;
2721
2722 /*LINTED*/
2723 buftagp = UMEM_BUFTAG(cp, buf);
2724
2725 /*
2726 * Read the buffer to check.
2727 */
2728 if (mdb_vread(buf, umv->umv_size, addr) == -1) {
2729 if (!besilent)
2730 mdb_warn("couldn't read %p", addr);
2731 return (WALK_NEXT);
2732 }
2733
2734 if ((corrupt = verify_pattern(buf, cp->cache_verify,
2735 UMEM_FREE_PATTERN)) >= 0) {
2736 if (!besilent)
2737 mdb_printf("buffer %p (free) seems corrupted, at %p\n",
2738 addr, (uintptr_t)addr + corrupt);
2739 goto corrupt;
2740 }
2741
2742 if ((cp->cache_flags & UMF_HASH) &&
2743 buftagp->bt_redzone != UMEM_REDZONE_PATTERN) {
2744 if (!besilent)
2745 mdb_printf("buffer %p (free) seems to "
2746 "have a corrupt redzone pattern\n", addr);
2747 goto corrupt;
2748 }
2749
2750 /*
2751 * confirm bufctl pointer integrity.
2752 */
2753 if (verify_buftag(buftagp, UMEM_BUFTAG_FREE) == -1) {
2754 if (!besilent)
2755 mdb_printf("buffer %p (free) has a corrupt "
2756 "buftag\n", addr);
2757 goto corrupt;
2758 }
2759
2760 return (WALK_NEXT);
2761 corrupt:
2762 umv->umv_corruption++;
2763 return (WALK_NEXT);
2764 }
2765
2766 /*
2767 * verify_alloc()
2768 * Verify that the buftag of an allocated buffer makes sense with respect
2769 * to the buffer.
2770 */
2771 /*ARGSUSED1*/
2772 static int
2773 verify_alloc(uintptr_t addr, const void *data, void *private)
2774 {
2775 umem_verify_t *umv = (umem_verify_t *)private;
2776 umem_cache_t *cp = &umv->umv_cache;
2777 uint64_t *buf = umv->umv_buf; /* buf to validate */
2778 /*LINTED*/
2779 umem_buftag_t *buftagp = UMEM_BUFTAG(cp, buf);
2780 uint32_t *ip = (uint32_t *)buftagp;
2781 uint8_t *bp = (uint8_t *)buf;
2782 int looks_ok = 0, size_ok = 1; /* flags for finding corruption */
2783 int besilent = umv->umv_besilent;
2784
2785 /*
2786 * Read the buffer to check.
2787 */
2788 if (mdb_vread(buf, umv->umv_size, addr) == -1) {
2789 if (!besilent)
2790 mdb_warn("couldn't read %p", addr);
2791 return (WALK_NEXT);
2792 }
2793
2794 /*
2795 * There are two cases to handle:
2796 * 1. If the buf was alloc'd using umem_cache_alloc, it will have
2797 * 0xfeedfacefeedface at the end of it
2798 * 2. If the buf was alloc'd using umem_alloc, it will have
2799 * 0xbb just past the end of the region in use. At the buftag,
2800 * it will have 0xfeedface (or, if the whole buffer is in use,
2801 * 0xfeedface & bb000000 or 0xfeedfacf & 000000bb depending on
2802 * endianness), followed by 32 bits containing the offset of the
2803 * 0xbb byte in the buffer.
2804 *
2805 * Finally, the two 32-bit words that comprise the second half of the
2806 * buftag should xor to UMEM_BUFTAG_ALLOC
2807 */
2808
2809 if (buftagp->bt_redzone == UMEM_REDZONE_PATTERN)
2810 looks_ok = 1;
2811 else if (!UMEM_SIZE_VALID(ip[1]))
2812 size_ok = 0;
2813 else if (bp[UMEM_SIZE_DECODE(ip[1])] == UMEM_REDZONE_BYTE)
2814 looks_ok = 1;
2815 else
2816 size_ok = 0;
2817
2818 if (!size_ok) {
2819 if (!besilent)
2820 mdb_printf("buffer %p (allocated) has a corrupt "
2821 "redzone size encoding\n", addr);
2822 goto corrupt;
2823 }
2824
2825 if (!looks_ok) {
2826 if (!besilent)
2827 mdb_printf("buffer %p (allocated) has a corrupt "
2828 "redzone signature\n", addr);
2829 goto corrupt;
2830 }
2831
2832 if (verify_buftag(buftagp, UMEM_BUFTAG_ALLOC) == -1) {
2833 if (!besilent)
2834 mdb_printf("buffer %p (allocated) has a "
2835 "corrupt buftag\n", addr);
2836 goto corrupt;
2837 }
2838
2839 return (WALK_NEXT);
2840 corrupt:
2841 umv->umv_corruption++;
2842 return (WALK_NEXT);
2843 }
2844
2845 /*ARGSUSED2*/
2846 int
2847 umem_verify(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
2848 {
2849 if (flags & DCMD_ADDRSPEC) {
2850 int check_alloc = 0, check_free = 0;
2851 umem_verify_t umv;
2852
2853 if (mdb_vread(&umv.umv_cache, sizeof (umv.umv_cache),
2854 addr) == -1) {
2855 mdb_warn("couldn't read umem_cache %p", addr);
2856 return (DCMD_ERR);
2857 }
2858
2859 umv.umv_size = umv.umv_cache.cache_buftag +
2860 sizeof (umem_buftag_t);
2861 umv.umv_buf = mdb_alloc(umv.umv_size, UM_SLEEP | UM_GC);
2862 umv.umv_corruption = 0;
2863
2864 if ((umv.umv_cache.cache_flags & UMF_REDZONE)) {
2865 check_alloc = 1;
2866 if (umv.umv_cache.cache_flags & UMF_DEADBEEF)
2867 check_free = 1;
2868 } else {
2869 if (!(flags & DCMD_LOOP)) {
2870 mdb_warn("cache %p (%s) does not have "
2871 "redzone checking enabled\n", addr,
2872 umv.umv_cache.cache_name);
2873 }
2874 return (DCMD_ERR);
2875 }
2876
2877 if (flags & DCMD_LOOP) {
2878 /*
2879 * table mode, don't print out every corrupt buffer
2880 */
2881 umv.umv_besilent = 1;
2882 } else {
2883 mdb_printf("Summary for cache '%s'\n",
2884 umv.umv_cache.cache_name);
2885 mdb_inc_indent(2);
2886 umv.umv_besilent = 0;
2887 }
2888
2889 if (check_alloc)
2890 (void) mdb_pwalk("umem", verify_alloc, &umv, addr);
2891 if (check_free)
2892 (void) mdb_pwalk("freemem", verify_free, &umv, addr);
2893
2894 if (flags & DCMD_LOOP) {
2895 if (umv.umv_corruption == 0) {
2896 mdb_printf("%-*s %?p clean\n",
2897 UMEM_CACHE_NAMELEN,
2898 umv.umv_cache.cache_name, addr);
2899 } else {
2900 char *s = ""; /* optional s in "buffer[s]" */
2901 if (umv.umv_corruption > 1)
2902 s = "s";
2903
2904 mdb_printf("%-*s %?p %d corrupt buffer%s\n",
2905 UMEM_CACHE_NAMELEN,
2906 umv.umv_cache.cache_name, addr,
2907 umv.umv_corruption, s);
2908 }
2909 } else {
2910 /*
2911 * This is the more verbose mode, when the user has
2912 * type addr::umem_verify. If the cache was clean,
2913 * nothing will have yet been printed. So say something.
2914 */
2915 if (umv.umv_corruption == 0)
2916 mdb_printf("clean\n");
2917
2918 mdb_dec_indent(2);
2919 }
2920 } else {
2921 /*
2922 * If the user didn't specify a cache to verify, we'll walk all
2923 * umem_cache's, specifying ourself as a callback for each...
2924 * this is the equivalent of '::walk umem_cache .::umem_verify'
2925 */
2926 mdb_printf("%<u>%-*s %-?s %-20s%</b>\n", UMEM_CACHE_NAMELEN,
2927 "Cache Name", "Addr", "Cache Integrity");
2928 (void) (mdb_walk_dcmd("umem_cache", "umem_verify", 0, NULL));
2929 }
2930
2931 return (DCMD_OK);
2932 }
2933
2934 typedef struct vmem_node {
2935 struct vmem_node *vn_next;
2936 struct vmem_node *vn_parent;
2937 struct vmem_node *vn_sibling;
2938 struct vmem_node *vn_children;
2939 uintptr_t vn_addr;
2940 int vn_marked;
2941 vmem_t vn_vmem;
2942 } vmem_node_t;
2943
2944 typedef struct vmem_walk {
2945 vmem_node_t *vw_root;
2946 vmem_node_t *vw_current;
2947 } vmem_walk_t;
2948
2949 int
2950 vmem_walk_init(mdb_walk_state_t *wsp)
2951 {
2952 uintptr_t vaddr, paddr;
2953 vmem_node_t *head = NULL, *root = NULL, *current = NULL, *parent, *vp;
2954 vmem_walk_t *vw;
2955
2956 if (umem_readvar(&vaddr, "vmem_list") == -1) {
2957 mdb_warn("couldn't read 'vmem_list'");
2958 return (WALK_ERR);
2959 }
2960
2961 while (vaddr != NULL) {
2962 vp = mdb_zalloc(sizeof (vmem_node_t), UM_SLEEP);
2963 vp->vn_addr = vaddr;
2964 vp->vn_next = head;
2965 head = vp;
2966
2967 if (vaddr == wsp->walk_addr)
2968 current = vp;
2969
2970 if (mdb_vread(&vp->vn_vmem, sizeof (vmem_t), vaddr) == -1) {
2971 mdb_warn("couldn't read vmem_t at %p", vaddr);
2972 goto err;
2973 }
2974
2975 vaddr = (uintptr_t)vp->vn_vmem.vm_next;
2976 }
2977
2978 for (vp = head; vp != NULL; vp = vp->vn_next) {
2979
2980 if ((paddr = (uintptr_t)vp->vn_vmem.vm_source) == NULL) {
2981 vp->vn_sibling = root;
2982 root = vp;
2983 continue;
2984 }
2985
2986 for (parent = head; parent != NULL; parent = parent->vn_next) {
2987 if (parent->vn_addr != paddr)
2988 continue;
2989 vp->vn_sibling = parent->vn_children;
2990 parent->vn_children = vp;
2991 vp->vn_parent = parent;
2992 break;
2993 }
2994
2995 if (parent == NULL) {
2996 mdb_warn("couldn't find %p's parent (%p)\n",
2997 vp->vn_addr, paddr);
2998 goto err;
2999 }
3000 }
3001
3002 vw = mdb_zalloc(sizeof (vmem_walk_t), UM_SLEEP);
3003 vw->vw_root = root;
3004
3005 if (current != NULL)
3006 vw->vw_current = current;
3007 else
3008 vw->vw_current = root;
3009
3010 wsp->walk_data = vw;
3011 return (WALK_NEXT);
3012 err:
3013 for (vp = head; head != NULL; vp = head) {
3014 head = vp->vn_next;
3015 mdb_free(vp, sizeof (vmem_node_t));
3016 }
3017
3018 return (WALK_ERR);
3019 }
3020
3021 int
3022 vmem_walk_step(mdb_walk_state_t *wsp)
3023 {
3024 vmem_walk_t *vw = wsp->walk_data;
3025 vmem_node_t *vp;
3026 int rval;
3027
3028 if ((vp = vw->vw_current) == NULL)
3029 return (WALK_DONE);
3030
3031 rval = wsp->walk_callback(vp->vn_addr, &vp->vn_vmem, wsp->walk_cbdata);
3032
3033 if (vp->vn_children != NULL) {
3034 vw->vw_current = vp->vn_children;
3035 return (rval);
3036 }
3037
3038 do {
3039 vw->vw_current = vp->vn_sibling;
3040 vp = vp->vn_parent;
3041 } while (vw->vw_current == NULL && vp != NULL);
3042
3043 return (rval);
3044 }
3045
3046 /*
3047 * The "vmem_postfix" walk walks the vmem arenas in post-fix order; all
3048 * children are visited before their parent. We perform the postfix walk
3049 * iteratively (rather than recursively) to allow mdb to regain control
3050 * after each callback.
3051 */
3052 int
3053 vmem_postfix_walk_step(mdb_walk_state_t *wsp)
3054 {
3055 vmem_walk_t *vw = wsp->walk_data;
3056 vmem_node_t *vp = vw->vw_current;
3057 int rval;
3058
3059 /*
3060 * If this node is marked, then we know that we have already visited
3061 * all of its children. If the node has any siblings, they need to
3062 * be visited next; otherwise, we need to visit the parent. Note
3063 * that vp->vn_marked will only be zero on the first invocation of
3064 * the step function.
3065 */
3066 if (vp->vn_marked) {
3067 if (vp->vn_sibling != NULL)
3068 vp = vp->vn_sibling;
3069 else if (vp->vn_parent != NULL)
3070 vp = vp->vn_parent;
3071 else {
3072 /*
3073 * We have neither a parent, nor a sibling, and we
3074 * have already been visited; we're done.
3075 */
3076 return (WALK_DONE);
3077 }
3078 }
3079
3080 /*
3081 * Before we visit this node, visit its children.
3082 */
3083 while (vp->vn_children != NULL && !vp->vn_children->vn_marked)
3084 vp = vp->vn_children;
3085
3086 vp->vn_marked = 1;
3087 vw->vw_current = vp;
3088 rval = wsp->walk_callback(vp->vn_addr, &vp->vn_vmem, wsp->walk_cbdata);
3089
3090 return (rval);
3091 }
3092
3093 void
3094 vmem_walk_fini(mdb_walk_state_t *wsp)
3095 {
3096 vmem_walk_t *vw = wsp->walk_data;
3097 vmem_node_t *root = vw->vw_root;
3098 int done;
3099
3100 if (root == NULL)
3101 return;
3102
3103 if ((vw->vw_root = root->vn_children) != NULL)
3104 vmem_walk_fini(wsp);
3105
3106 vw->vw_root = root->vn_sibling;
3107 done = (root->vn_sibling == NULL && root->vn_parent == NULL);
3108 mdb_free(root, sizeof (vmem_node_t));
3109
3110 if (done) {
3111 mdb_free(vw, sizeof (vmem_walk_t));
3112 } else {
3113 vmem_walk_fini(wsp);
3114 }
3115 }
3116
3117 typedef struct vmem_seg_walk {
3118 uint8_t vsw_type;
3119 uintptr_t vsw_start;
3120 uintptr_t vsw_current;
3121 } vmem_seg_walk_t;
3122
3123 /*ARGSUSED*/
3124 int
3125 vmem_seg_walk_common_init(mdb_walk_state_t *wsp, uint8_t type, char *name)
3126 {
3127 vmem_seg_walk_t *vsw;
3128
3129 if (wsp->walk_addr == NULL) {
3130 mdb_warn("vmem_%s does not support global walks\n", name);
3131 return (WALK_ERR);
3132 }
3133
3134 wsp->walk_data = vsw = mdb_alloc(sizeof (vmem_seg_walk_t), UM_SLEEP);
3135
3136 vsw->vsw_type = type;
3137 vsw->vsw_start = wsp->walk_addr + OFFSETOF(vmem_t, vm_seg0);
3138 vsw->vsw_current = vsw->vsw_start;
3139
3140 return (WALK_NEXT);
3141 }
3142
3143 /*
3144 * vmem segments can't have type 0 (this should be added to vmem_impl.h).
3145 */
3146 #define VMEM_NONE 0
3147
3148 int
3149 vmem_alloc_walk_init(mdb_walk_state_t *wsp)
3150 {
3151 return (vmem_seg_walk_common_init(wsp, VMEM_ALLOC, "alloc"));
3152 }
3153
3154 int
3155 vmem_free_walk_init(mdb_walk_state_t *wsp)
3156 {
3157 return (vmem_seg_walk_common_init(wsp, VMEM_FREE, "free"));
3158 }
3159
3160 int
3161 vmem_span_walk_init(mdb_walk_state_t *wsp)
3162 {
3163 return (vmem_seg_walk_common_init(wsp, VMEM_SPAN, "span"));
3164 }
3165
3166 int
3167 vmem_seg_walk_init(mdb_walk_state_t *wsp)
3168 {
3169 return (vmem_seg_walk_common_init(wsp, VMEM_NONE, "seg"));
3170 }
3171
3172 int
3173 vmem_seg_walk_step(mdb_walk_state_t *wsp)
3174 {
3175 vmem_seg_t seg;
3176 vmem_seg_walk_t *vsw = wsp->walk_data;
3177 uintptr_t addr = vsw->vsw_current;
3178 static size_t seg_size = 0;
3179 int rval;
3180
3181 if (!seg_size) {
3182 if (umem_readvar(&seg_size, "vmem_seg_size") == -1) {
3183 mdb_warn("failed to read 'vmem_seg_size'");
3184 seg_size = sizeof (vmem_seg_t);
3185 }
3186 }
3187
3188 if (seg_size < sizeof (seg))
3189 bzero((caddr_t)&seg + seg_size, sizeof (seg) - seg_size);
3190
3191 if (mdb_vread(&seg, seg_size, addr) == -1) {
3192 mdb_warn("couldn't read vmem_seg at %p", addr);
3193 return (WALK_ERR);
3194 }
3195
3196 vsw->vsw_current = (uintptr_t)seg.vs_anext;
3197 if (vsw->vsw_type != VMEM_NONE && seg.vs_type != vsw->vsw_type) {
3198 rval = WALK_NEXT;
3199 } else {
3200 rval = wsp->walk_callback(addr, &seg, wsp->walk_cbdata);
3201 }
3202
3203 if (vsw->vsw_current == vsw->vsw_start)
3204 return (WALK_DONE);
3205
3206 return (rval);
3207 }
3208
3209 void
3210 vmem_seg_walk_fini(mdb_walk_state_t *wsp)
3211 {
3212 vmem_seg_walk_t *vsw = wsp->walk_data;
3213
3214 mdb_free(vsw, sizeof (vmem_seg_walk_t));
3215 }
3216
3217 #define VMEM_NAMEWIDTH 22
3218
3219 int
3220 vmem(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
3221 {
3222 vmem_t v, parent;
3223 uintptr_t paddr;
3224 int ident = 0;
3225 char c[VMEM_NAMEWIDTH];
3226
3227 if (!(flags & DCMD_ADDRSPEC)) {
3228 if (mdb_walk_dcmd("vmem", "vmem", argc, argv) == -1) {
3229 mdb_warn("can't walk vmem");
3230 return (DCMD_ERR);
3231 }
3232 return (DCMD_OK);
3233 }
3234
3235 if (DCMD_HDRSPEC(flags))
3236 mdb_printf("%-?s %-*s %10s %12s %9s %5s\n",
3237 "ADDR", VMEM_NAMEWIDTH, "NAME", "INUSE",
3238 "TOTAL", "SUCCEED", "FAIL");
3239
3240 if (mdb_vread(&v, sizeof (v), addr) == -1) {
3241 mdb_warn("couldn't read vmem at %p", addr);
3242 return (DCMD_ERR);
3243 }
3244
3245 for (paddr = (uintptr_t)v.vm_source; paddr != NULL; ident += 2) {
3246 if (mdb_vread(&parent, sizeof (parent), paddr) == -1) {
3247 mdb_warn("couldn't trace %p's ancestry", addr);
3248 ident = 0;
3249 break;
3250 }
3251 paddr = (uintptr_t)parent.vm_source;
3252 }
3253
3254 (void) mdb_snprintf(c, VMEM_NAMEWIDTH, "%*s%s", ident, "", v.vm_name);
3255
3256 mdb_printf("%0?p %-*s %10llu %12llu %9llu %5llu\n",
3257 addr, VMEM_NAMEWIDTH, c,
3258 v.vm_kstat.vk_mem_inuse, v.vm_kstat.vk_mem_total,
3259 v.vm_kstat.vk_alloc, v.vm_kstat.vk_fail);
3260
3261 return (DCMD_OK);
3262 }
3263
3264 void
3265 vmem_seg_help(void)
3266 {
3267 mdb_printf("%s\n",
3268 "Display the contents of vmem_seg_ts, with optional filtering.\n"
3269 "\n"
3270 "A vmem_seg_t represents a range of addresses (or arbitrary numbers),\n"
3271 "representing a single chunk of data. Only ALLOC segments have debugging\n"
3272 "information.\n");
3273 mdb_dec_indent(2);
3274 mdb_printf("%<b>OPTIONS%</b>\n");
3275 mdb_inc_indent(2);
3276 mdb_printf("%s",
3277 " -v Display the full content of the vmem_seg, including its stack trace\n"
3278 " -s report the size of the segment, instead of the end address\n"
3279 " -c caller\n"
3280 " filter out segments without the function/PC in their stack trace\n"
3281 " -e earliest\n"
3282 " filter out segments timestamped before earliest\n"
3283 " -l latest\n"
3284 " filter out segments timestamped after latest\n"
3285 " -m minsize\n"
3286 " filer out segments smaller than minsize\n"
3287 " -M maxsize\n"
3288 " filer out segments larger than maxsize\n"
3289 " -t thread\n"
3290 " filter out segments not involving thread\n"
3291 " -T type\n"
3292 " filter out segments not of type 'type'\n"
3293 " type is one of: ALLOC/FREE/SPAN/ROTOR/WALKER\n");
3294 }
3295
3296
3297 /*ARGSUSED*/
3298 int
3299 vmem_seg(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
3300 {
3301 vmem_seg_t vs;
3302 uintptr_t *stk = vs.vs_stack;
3303 uintptr_t sz;
3304 uint8_t t;
3305 const char *type = NULL;
3306 GElf_Sym sym;
3307 char c[MDB_SYM_NAMLEN];
3308 int no_debug;
3309 int i;
3310 int depth;
3311 uintptr_t laddr, haddr;
3312
3313 uintptr_t caller = NULL, thread = NULL;
3314 uintptr_t minsize = 0, maxsize = 0;
3315
3316 hrtime_t earliest = 0, latest = 0;
3317
3318 uint_t size = 0;
3319 uint_t verbose = 0;
3320
3321 if (!(flags & DCMD_ADDRSPEC))
3322 return (DCMD_USAGE);
3323
3324 if (mdb_getopts(argc, argv,
3325 'c', MDB_OPT_UINTPTR, &caller,
3326 'e', MDB_OPT_UINT64, &earliest,
3327 'l', MDB_OPT_UINT64, &latest,
3328 's', MDB_OPT_SETBITS, TRUE, &size,
3329 'm', MDB_OPT_UINTPTR, &minsize,
3330 'M', MDB_OPT_UINTPTR, &maxsize,
3331 't', MDB_OPT_UINTPTR, &thread,
3332 'T', MDB_OPT_STR, &type,
3333 'v', MDB_OPT_SETBITS, TRUE, &verbose,
3334 NULL) != argc)
3335 return (DCMD_USAGE);
3336
3337 if (DCMD_HDRSPEC(flags) && !(flags & DCMD_PIPE_OUT)) {
3338 if (verbose) {
3339 mdb_printf("%16s %4s %16s %16s %16s\n"
3340 "%<u>%16s %4s %16s %16s %16s%</u>\n",
3341 "ADDR", "TYPE", "START", "END", "SIZE",
3342 "", "", "THREAD", "TIMESTAMP", "");
3343 } else {
3344 mdb_printf("%?s %4s %?s %?s %s\n", "ADDR", "TYPE",
3345 "START", size? "SIZE" : "END", "WHO");
3346 }
3347 }
3348
3349 if (mdb_vread(&vs, sizeof (vs), addr) == -1) {
3350 mdb_warn("couldn't read vmem_seg at %p", addr);
3351 return (DCMD_ERR);
3352 }
3353
3354 if (type != NULL) {
3355 if (strcmp(type, "ALLC") == 0 || strcmp(type, "ALLOC") == 0)
3356 t = VMEM_ALLOC;
3357 else if (strcmp(type, "FREE") == 0)
3358 t = VMEM_FREE;
3359 else if (strcmp(type, "SPAN") == 0)
3360 t = VMEM_SPAN;
3361 else if (strcmp(type, "ROTR") == 0 ||
3362 strcmp(type, "ROTOR") == 0)
3363 t = VMEM_ROTOR;
3364 else if (strcmp(type, "WLKR") == 0 ||
3365 strcmp(type, "WALKER") == 0)
3366 t = VMEM_WALKER;
3367 else {
3368 mdb_warn("\"%s\" is not a recognized vmem_seg type\n",
3369 type);
3370 return (DCMD_ERR);
3371 }
3372
3373 if (vs.vs_type != t)
3374 return (DCMD_OK);
3375 }
3376
3377 sz = vs.vs_end - vs.vs_start;
3378
3379 if (minsize != 0 && sz < minsize)
3380 return (DCMD_OK);
3381
3382 if (maxsize != 0 && sz > maxsize)
3383 return (DCMD_OK);
3384
3385 t = vs.vs_type;
3386 depth = vs.vs_depth;
3387
3388 /*
3389 * debug info, when present, is only accurate for VMEM_ALLOC segments
3390 */
3391 no_debug = (t != VMEM_ALLOC) ||
3392 (depth == 0 || depth > VMEM_STACK_DEPTH);
3393
3394 if (no_debug) {
3395 if (caller != NULL || thread != NULL || earliest != 0 ||
3396 latest != 0)
3397 return (DCMD_OK); /* not enough info */
3398 } else {
3399 if (caller != NULL) {
3400 laddr = caller;
3401 haddr = caller + sizeof (caller);
3402
3403 if (mdb_lookup_by_addr(caller, MDB_SYM_FUZZY, c,
3404 sizeof (c), &sym) != -1 &&
3405 caller == (uintptr_t)sym.st_value) {
3406 /*
3407 * We were provided an exact symbol value; any
3408 * address in the function is valid.
3409 */
3410 laddr = (uintptr_t)sym.st_value;
3411 haddr = (uintptr_t)sym.st_value + sym.st_size;
3412 }
3413
3414 for (i = 0; i < depth; i++)
3415 if (vs.vs_stack[i] >= laddr &&
3416 vs.vs_stack[i] < haddr)
3417 break;
3418
3419 if (i == depth)
3420 return (DCMD_OK);
3421 }
3422
3423 if (thread != NULL && (uintptr_t)vs.vs_thread != thread)
3424 return (DCMD_OK);
3425
3426 if (earliest != 0 && vs.vs_timestamp < earliest)
3427 return (DCMD_OK);
3428
3429 if (latest != 0 && vs.vs_timestamp > latest)
3430 return (DCMD_OK);
3431 }
3432
3433 type = (t == VMEM_ALLOC ? "ALLC" :
3434 t == VMEM_FREE ? "FREE" :
3435 t == VMEM_SPAN ? "SPAN" :
3436 t == VMEM_ROTOR ? "ROTR" :
3437 t == VMEM_WALKER ? "WLKR" :
3438 "????");
3439
3440 if (flags & DCMD_PIPE_OUT) {
3441 mdb_printf("%#r\n", addr);
3442 return (DCMD_OK);
3443 }
3444
3445 if (verbose) {
3446 mdb_printf("%<b>%16p%</b> %4s %16p %16p %16d\n",
3447 addr, type, vs.vs_start, vs.vs_end, sz);
3448
3449 if (no_debug)
3450 return (DCMD_OK);
3451
3452 mdb_printf("%16s %4s %16d %16llx\n",
3453 "", "", vs.vs_thread, vs.vs_timestamp);
3454
3455 mdb_inc_indent(17);
3456 for (i = 0; i < depth; i++) {
3457 mdb_printf("%a\n", stk[i]);
3458 }
3459 mdb_dec_indent(17);
3460 mdb_printf("\n");
3461 } else {
3462 mdb_printf("%0?p %4s %0?p %0?p", addr, type,
3463 vs.vs_start, size? sz : vs.vs_end);
3464
3465 if (no_debug) {
3466 mdb_printf("\n");
3467 return (DCMD_OK);
3468 }
3469
3470 for (i = 0; i < depth; i++) {
3471 if (mdb_lookup_by_addr(stk[i], MDB_SYM_FUZZY,
3472 c, sizeof (c), &sym) == -1)
3473 continue;
3474 if (is_umem_sym(c, "vmem_"))
3475 continue;
3476 break;
3477 }
3478 mdb_printf(" %a\n", stk[i]);
3479 }
3480 return (DCMD_OK);
3481 }
3482
3483 /*ARGSUSED*/
3484 static int
3485 showbc(uintptr_t addr, const umem_bufctl_audit_t *bcp, hrtime_t *newest)
3486 {
3487 char name[UMEM_CACHE_NAMELEN + 1];
3488 hrtime_t delta;
3489 int i, depth;
3490
3491 if (bcp->bc_timestamp == 0)
3492 return (WALK_DONE);
3493
3494 if (*newest == 0)
3495 *newest = bcp->bc_timestamp;
3496
3497 delta = *newest - bcp->bc_timestamp;
3498 depth = MIN(bcp->bc_depth, umem_stack_depth);
3499
3500 if (mdb_readstr(name, sizeof (name), (uintptr_t)
3501 &bcp->bc_cache->cache_name) <= 0)
3502 (void) mdb_snprintf(name, sizeof (name), "%a", bcp->bc_cache);
3503
3504 mdb_printf("\nT-%lld.%09lld addr=%p %s\n",
3505 delta / NANOSEC, delta % NANOSEC, bcp->bc_addr, name);
3506
3507 for (i = 0; i < depth; i++)
3508 mdb_printf("\t %a\n", bcp->bc_stack[i]);
3509
3510 return (WALK_NEXT);
3511 }
3512
3513 int
3514 umalog(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
3515 {
3516 const char *logname = "umem_transaction_log";
3517 hrtime_t newest = 0;
3518
3519 if ((flags & DCMD_ADDRSPEC) || argc > 1)
3520 return (DCMD_USAGE);
3521
3522 if (argc > 0) {
3523 if (argv->a_type != MDB_TYPE_STRING)
3524 return (DCMD_USAGE);
3525 if (strcmp(argv->a_un.a_str, "fail") == 0)
3526 logname = "umem_failure_log";
3527 else if (strcmp(argv->a_un.a_str, "slab") == 0)
3528 logname = "umem_slab_log";
3529 else
3530 return (DCMD_USAGE);
3531 }
3532
3533 if (umem_readvar(&addr, logname) == -1) {
3534 mdb_warn("failed to read %s log header pointer");
3535 return (DCMD_ERR);
3536 }
3537
3538 if (mdb_pwalk("umem_log", (mdb_walk_cb_t)showbc, &newest, addr) == -1) {
3539 mdb_warn("failed to walk umem log");
3540 return (DCMD_ERR);
3541 }
3542
3543 return (DCMD_OK);
3544 }
3545
3546 /*
3547 * As the final lure for die-hard crash(1M) users, we provide ::umausers here.
3548 * The first piece is a structure which we use to accumulate umem_cache_t
3549 * addresses of interest. The umc_add is used as a callback for the umem_cache
3550 * walker; we either add all caches, or ones named explicitly as arguments.
3551 */
3552
3553 typedef struct umclist {
3554 const char *umc_name; /* Name to match (or NULL) */
3555 uintptr_t *umc_caches; /* List of umem_cache_t addrs */
3556 int umc_nelems; /* Num entries in umc_caches */
3557 int umc_size; /* Size of umc_caches array */
3558 } umclist_t;
3559
3560 static int
3561 umc_add(uintptr_t addr, const umem_cache_t *cp, umclist_t *umc)
3562 {
3563 void *p;
3564 int s;
3565
3566 if (umc->umc_name == NULL ||
3567 strcmp(cp->cache_name, umc->umc_name) == 0) {
3568 /*
3569 * If we have a match, grow our array (if necessary), and then
3570 * add the virtual address of the matching cache to our list.
3571 */
3572 if (umc->umc_nelems >= umc->umc_size) {
3573 s = umc->umc_size ? umc->umc_size * 2 : 256;
3574 p = mdb_alloc(sizeof (uintptr_t) * s, UM_SLEEP | UM_GC);
3575
3576 bcopy(umc->umc_caches, p,
3577 sizeof (uintptr_t) * umc->umc_size);
3578
3579 umc->umc_caches = p;
3580 umc->umc_size = s;
3581 }
3582
3583 umc->umc_caches[umc->umc_nelems++] = addr;
3584 return (umc->umc_name ? WALK_DONE : WALK_NEXT);
3585 }
3586
3587 return (WALK_NEXT);
3588 }
3589
3590 /*
3591 * The second piece of ::umausers is a hash table of allocations. Each
3592 * allocation owner is identified by its stack trace and data_size. We then
3593 * track the total bytes of all such allocations, and the number of allocations
3594 * to report at the end. Once we have a list of caches, we walk through the
3595 * allocated bufctls of each, and update our hash table accordingly.
3596 */
3597
3598 typedef struct umowner {
3599 struct umowner *umo_head; /* First hash elt in bucket */
3600 struct umowner *umo_next; /* Next hash elt in chain */
3601 size_t umo_signature; /* Hash table signature */
3602 uint_t umo_num; /* Number of allocations */
3603 size_t umo_data_size; /* Size of each allocation */
3604 size_t umo_total_size; /* Total bytes of allocation */
3605 int umo_depth; /* Depth of stack trace */
3606 uintptr_t *umo_stack; /* Stack trace */
3607 } umowner_t;
3608
3609 typedef struct umusers {
3610 const umem_cache_t *umu_cache; /* Current umem cache */
3611 umowner_t *umu_hash; /* Hash table of owners */
3612 uintptr_t *umu_stacks; /* stacks for owners */
3613 int umu_nelems; /* Number of entries in use */
3614 int umu_size; /* Total number of entries */
3615 } umusers_t;
3616
3617 static void
3618 umu_add(umusers_t *umu, const umem_bufctl_audit_t *bcp,
3619 size_t size, size_t data_size)
3620 {
3621 int i, depth = MIN(bcp->bc_depth, umem_stack_depth);
3622 size_t bucket, signature = data_size;
3623 umowner_t *umo, *umoend;
3624
3625 /*
3626 * If the hash table is full, double its size and rehash everything.
3627 */
3628 if (umu->umu_nelems >= umu->umu_size) {
3629 int s = umu->umu_size ? umu->umu_size * 2 : 1024;
3630 size_t umowner_size = sizeof (umowner_t);
3631 size_t trace_size = umem_stack_depth * sizeof (uintptr_t);
3632 uintptr_t *new_stacks;
3633
3634 umo = mdb_alloc(umowner_size * s, UM_SLEEP | UM_GC);
3635 new_stacks = mdb_alloc(trace_size * s, UM_SLEEP | UM_GC);
3636
3637 bcopy(umu->umu_hash, umo, umowner_size * umu->umu_size);
3638 bcopy(umu->umu_stacks, new_stacks, trace_size * umu->umu_size);
3639 umu->umu_hash = umo;
3640 umu->umu_stacks = new_stacks;
3641 umu->umu_size = s;
3642
3643 umoend = umu->umu_hash + umu->umu_size;
3644 for (umo = umu->umu_hash; umo < umoend; umo++) {
3645 umo->umo_head = NULL;
3646 umo->umo_stack = &umu->umu_stacks[
3647 umem_stack_depth * (umo - umu->umu_hash)];
3648 }
3649
3650 umoend = umu->umu_hash + umu->umu_nelems;
3651 for (umo = umu->umu_hash; umo < umoend; umo++) {
3652 bucket = umo->umo_signature & (umu->umu_size - 1);
3653 umo->umo_next = umu->umu_hash[bucket].umo_head;
3654 umu->umu_hash[bucket].umo_head = umo;
3655 }
3656 }
3657
3658 /*
3659 * Finish computing the hash signature from the stack trace, and then
3660 * see if the owner is in the hash table. If so, update our stats.
3661 */
3662 for (i = 0; i < depth; i++)
3663 signature += bcp->bc_stack[i];
3664
3665 bucket = signature & (umu->umu_size - 1);
3666
3667 for (umo = umu->umu_hash[bucket].umo_head; umo; umo = umo->umo_next) {
3668 if (umo->umo_signature == signature) {
3669 size_t difference = 0;
3670
3671 difference |= umo->umo_data_size - data_size;
3672 difference |= umo->umo_depth - depth;
3673
3674 for (i = 0; i < depth; i++) {
3675 difference |= umo->umo_stack[i] -
3676 bcp->bc_stack[i];
3677 }
3678
3679 if (difference == 0) {
3680 umo->umo_total_size += size;
3681 umo->umo_num++;
3682 return;
3683 }
3684 }
3685 }
3686
3687 /*
3688 * If the owner is not yet hashed, grab the next element and fill it
3689 * in based on the allocation information.
3690 */
3691 umo = &umu->umu_hash[umu->umu_nelems++];
3692 umo->umo_next = umu->umu_hash[bucket].umo_head;
3693 umu->umu_hash[bucket].umo_head = umo;
3694
3695 umo->umo_signature = signature;
3696 umo->umo_num = 1;
3697 umo->umo_data_size = data_size;
3698 umo->umo_total_size = size;
3699 umo->umo_depth = depth;
3700
3701 for (i = 0; i < depth; i++)
3702 umo->umo_stack[i] = bcp->bc_stack[i];
3703 }
3704
3705 /*
3706 * When ::umausers is invoked without the -f flag, we simply update our hash
3707 * table with the information from each allocated bufctl.
3708 */
3709 /*ARGSUSED*/
3710 static int
3711 umause1(uintptr_t addr, const umem_bufctl_audit_t *bcp, umusers_t *umu)
3712 {
3713 const umem_cache_t *cp = umu->umu_cache;
3714
3715 umu_add(umu, bcp, cp->cache_bufsize, cp->cache_bufsize);
3716 return (WALK_NEXT);
3717 }
3718
3719 /*
3720 * When ::umausers is invoked with the -f flag, we print out the information
3721 * for each bufctl as well as updating the hash table.
3722 */
3723 static int
3724 umause2(uintptr_t addr, const umem_bufctl_audit_t *bcp, umusers_t *umu)
3725 {
3726 int i, depth = MIN(bcp->bc_depth, umem_stack_depth);
3727 const umem_cache_t *cp = umu->umu_cache;
3728
3729 mdb_printf("size %d, addr %p, thread %p, cache %s\n",
3730 cp->cache_bufsize, addr, bcp->bc_thread, cp->cache_name);
3731
3732 for (i = 0; i < depth; i++)
3733 mdb_printf("\t %a\n", bcp->bc_stack[i]);
3734
3735 umu_add(umu, bcp, cp->cache_bufsize, cp->cache_bufsize);
3736 return (WALK_NEXT);
3737 }
3738
3739 /*
3740 * We sort our results by allocation size before printing them.
3741 */
3742 static int
3743 umownercmp(const void *lp, const void *rp)
3744 {
3745 const umowner_t *lhs = lp;
3746 const umowner_t *rhs = rp;
3747
3748 return (rhs->umo_total_size - lhs->umo_total_size);
3749 }
3750
3751 /*
3752 * The main engine of ::umausers is relatively straightforward: First we
3753 * accumulate our list of umem_cache_t addresses into the umclist_t. Next we
3754 * iterate over the allocated bufctls of each cache in the list. Finally,
3755 * we sort and print our results.
3756 */
3757 /*ARGSUSED*/
3758 int
3759 umausers(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
3760 {
3761 int mem_threshold = 8192; /* Minimum # bytes for printing */
3762 int cnt_threshold = 100; /* Minimum # blocks for printing */
3763 int audited_caches = 0; /* Number of UMF_AUDIT caches found */
3764 int do_all_caches = 1; /* Do all caches (no arguments) */
3765 int opt_e = FALSE; /* Include "small" users */
3766 int opt_f = FALSE; /* Print stack traces */
3767
3768 mdb_walk_cb_t callback = (mdb_walk_cb_t)umause1;
3769 umowner_t *umo, *umoend;
3770 int i, oelems;
3771
3772 umclist_t umc;
3773 umusers_t umu;
3774
3775 if (flags & DCMD_ADDRSPEC)
3776 return (DCMD_USAGE);
3777
3778 bzero(&umc, sizeof (umc));
3779 bzero(&umu, sizeof (umu));
3780
3781 while ((i = mdb_getopts(argc, argv,
3782 'e', MDB_OPT_SETBITS, TRUE, &opt_e,
3783 'f', MDB_OPT_SETBITS, TRUE, &opt_f, NULL)) != argc) {
3784
3785 argv += i; /* skip past options we just processed */
3786 argc -= i; /* adjust argc */
3787
3788 if (argv->a_type != MDB_TYPE_STRING || *argv->a_un.a_str == '-')
3789 return (DCMD_USAGE);
3790
3791 oelems = umc.umc_nelems;
3792 umc.umc_name = argv->a_un.a_str;
3793 (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umc_add, &umc);
3794
3795 if (umc.umc_nelems == oelems) {
3796 mdb_warn("unknown umem cache: %s\n", umc.umc_name);
3797 return (DCMD_ERR);
3798 }
3799
3800 do_all_caches = 0;
3801 argv++;
3802 argc--;
3803 }
3804
3805 if (opt_e)
3806 mem_threshold = cnt_threshold = 0;
3807
3808 if (opt_f)
3809 callback = (mdb_walk_cb_t)umause2;
3810
3811 if (do_all_caches) {
3812 umc.umc_name = NULL; /* match all cache names */
3813 (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umc_add, &umc);
3814 }
3815
3816 for (i = 0; i < umc.umc_nelems; i++) {
3817 uintptr_t cp = umc.umc_caches[i];
3818 umem_cache_t c;
3819
3820 if (mdb_vread(&c, sizeof (c), cp) == -1) {
3821 mdb_warn("failed to read cache at %p", cp);
3822 continue;
3823 }
3824
3825 if (!(c.cache_flags & UMF_AUDIT)) {
3826 if (!do_all_caches) {
3827 mdb_warn("UMF_AUDIT is not enabled for %s\n",
3828 c.cache_name);
3829 }
3830 continue;
3831 }
3832
3833 umu.umu_cache = &c;
3834 (void) mdb_pwalk("bufctl", callback, &umu, cp);
3835 audited_caches++;
3836 }
3837
3838 if (audited_caches == 0 && do_all_caches) {
3839 mdb_warn("UMF_AUDIT is not enabled for any caches\n");
3840 return (DCMD_ERR);
3841 }
3842
3843 qsort(umu.umu_hash, umu.umu_nelems, sizeof (umowner_t), umownercmp);
3844 umoend = umu.umu_hash + umu.umu_nelems;
3845
3846 for (umo = umu.umu_hash; umo < umoend; umo++) {
3847 if (umo->umo_total_size < mem_threshold &&
3848 umo->umo_num < cnt_threshold)
3849 continue;
3850 mdb_printf("%lu bytes for %u allocations with data size %lu:\n",
3851 umo->umo_total_size, umo->umo_num, umo->umo_data_size);
3852 for (i = 0; i < umo->umo_depth; i++)
3853 mdb_printf("\t %a\n", umo->umo_stack[i]);
3854 }
3855
3856 return (DCMD_OK);
3857 }
3858
3859 struct malloc_data {
3860 uint32_t malloc_size;
3861 uint32_t malloc_stat; /* == UMEM_MALLOC_ENCODE(state, malloc_size) */
3862 };
3863
3864 #ifdef _LP64
3865 #define UMI_MAX_BUCKET (UMEM_MAXBUF - 2*sizeof (struct malloc_data))
3866 #else
3867 #define UMI_MAX_BUCKET (UMEM_MAXBUF - sizeof (struct malloc_data))
3868 #endif
3869
3870 typedef struct umem_malloc_info {
3871 size_t um_total; /* total allocated buffers */
3872 size_t um_malloc; /* malloc buffers */
3873 size_t um_malloc_size; /* sum of malloc buffer sizes */
3874 size_t um_malloc_overhead; /* sum of in-chunk overheads */
3875
3876 umem_cache_t *um_cp;
3877
3878 uint_t *um_bucket;
3879 } umem_malloc_info_t;
3880
3881 static void
3882 umem_malloc_print_dist(uint_t *um_bucket, size_t minmalloc, size_t maxmalloc,
3883 size_t maxbuckets, size_t minbucketsize, int geometric)
3884 {
3885 uint64_t um_malloc;
3886 int minb = -1;
3887 int maxb = -1;
3888 int buckets;
3889 int nbucks;
3890 int i;
3891 int b;
3892 const int *distarray;
3893
3894 minb = (int)minmalloc;
3895 maxb = (int)maxmalloc;
3896
3897 nbucks = buckets = maxb - minb + 1;
3898
3899 um_malloc = 0;
3900 for (b = minb; b <= maxb; b++)
3901 um_malloc += um_bucket[b];
3902
3903 if (maxbuckets != 0)
3904 buckets = MIN(buckets, maxbuckets);
3905
3906 if (minbucketsize > 1) {
3907 buckets = MIN(buckets, nbucks/minbucketsize);
3908 if (buckets == 0) {
3909 buckets = 1;
3910 minbucketsize = nbucks;
3911 }
3912 }
3913
3914 if (geometric)
3915 distarray = dist_geometric(buckets, minb, maxb, minbucketsize);
3916 else
3917 distarray = dist_linear(buckets, minb, maxb);
3918
3919 dist_print_header("malloc size", 11, "count");
3920 for (i = 0; i < buckets; i++) {
3921 dist_print_bucket(distarray, i, um_bucket, um_malloc, 11);
3922 }
3923 mdb_printf("\n");
3924 }
3925
3926 /*
3927 * A malloc()ed buffer looks like:
3928 *
3929 * <----------- mi.malloc_size --->
3930 * <----------- cp.cache_bufsize ------------------>
3931 * <----------- cp.cache_chunksize -------------------------------->
3932 * +-------+-----------------------+---------------+---------------+
3933 * |/tag///| mallocsz |/round-off/////|/debug info////|
3934 * +-------+---------------------------------------+---------------+
3935 * <-- usable space ------>
3936 *
3937 * mallocsz is the argument to malloc(3C).
3938 * mi.malloc_size is the actual size passed to umem_alloc(), which
3939 * is rounded up to the smallest available cache size, which is
3940 * cache_bufsize. If there is debugging or alignment overhead in
3941 * the cache, that is reflected in a larger cache_chunksize.
3942 *
3943 * The tag at the beginning of the buffer is either 8-bytes or 16-bytes,
3944 * depending upon the ISA's alignment requirements. For 32-bit allocations,
3945 * it is always a 8-byte tag. For 64-bit allocations larger than 8 bytes,
3946 * the tag has 8 bytes of padding before it.
3947 *
3948 * 32-byte, 64-byte buffers <= 8 bytes:
3949 * +-------+-------+--------- ...
3950 * |/size//|/stat//| mallocsz ...
3951 * +-------+-------+--------- ...
3952 * ^
3953 * pointer returned from malloc(3C)
3954 *
3955 * 64-byte buffers > 8 bytes:
3956 * +---------------+-------+-------+--------- ...
3957 * |/padding///////|/size//|/stat//| mallocsz ...
3958 * +---------------+-------+-------+--------- ...
3959 * ^
3960 * pointer returned from malloc(3C)
3961 *
3962 * The "size" field is "malloc_size", which is mallocsz + the padding.
3963 * The "stat" field is derived from malloc_size, and functions as a
3964 * validation that this buffer is actually from malloc(3C).
3965 */
3966 /*ARGSUSED*/
3967 static int
3968 um_umem_buffer_cb(uintptr_t addr, void *buf, umem_malloc_info_t *ump)
3969 {
3970 struct malloc_data md;
3971 size_t m_addr = addr;
3972 size_t overhead = sizeof (md);
3973 size_t mallocsz;
3974
3975 ump->um_total++;
3976
3977 #ifdef _LP64
3978 if (ump->um_cp->cache_bufsize > UMEM_SECOND_ALIGN) {
3979 m_addr += overhead;
3980 overhead += sizeof (md);
3981 }
3982 #endif
3983
3984 if (mdb_vread(&md, sizeof (md), m_addr) == -1) {
3985 mdb_warn("unable to read malloc header at %p", m_addr);
3986 return (WALK_NEXT);
3987 }
3988
3989 switch (UMEM_MALLOC_DECODE(md.malloc_stat, md.malloc_size)) {
3990 case MALLOC_MAGIC:
3991 #ifdef _LP64
3992 case MALLOC_SECOND_MAGIC:
3993 #endif
3994 mallocsz = md.malloc_size - overhead;
3995
3996 ump->um_malloc++;
3997 ump->um_malloc_size += mallocsz;
3998 ump->um_malloc_overhead += overhead;
3999
4000 /* include round-off and debug overhead */
4001 ump->um_malloc_overhead +=
4002 ump->um_cp->cache_chunksize - md.malloc_size;
4003
4004 if (ump->um_bucket != NULL && mallocsz <= UMI_MAX_BUCKET)
4005 ump->um_bucket[mallocsz]++;
4006
4007 break;
4008 default:
4009 break;
4010 }
4011
4012 return (WALK_NEXT);
4013 }
4014
4015 int
4016 get_umem_alloc_sizes(int **out, size_t *out_num)
4017 {
4018 GElf_Sym sym;
4019
4020 if (umem_lookup_by_name("umem_alloc_sizes", &sym) == -1) {
4021 mdb_warn("unable to look up umem_alloc_sizes");
4022 return (-1);
4023 }
4024
4025 *out = mdb_alloc(sym.st_size, UM_SLEEP | UM_GC);
4026 *out_num = sym.st_size / sizeof (int);
4027
4028 if (mdb_vread(*out, sym.st_size, sym.st_value) == -1) {
4029 mdb_warn("unable to read umem_alloc_sizes (%p)", sym.st_value);
4030 *out = NULL;
4031 return (-1);
4032 }
4033
4034 return (0);
4035 }
4036
4037
4038 static int
4039 um_umem_cache_cb(uintptr_t addr, umem_cache_t *cp, umem_malloc_info_t *ump)
4040 {
4041 if (strncmp(cp->cache_name, "umem_alloc_", strlen("umem_alloc_")) != 0)
4042 return (WALK_NEXT);
4043
4044 ump->um_cp = cp;
4045
4046 if (mdb_pwalk("umem", (mdb_walk_cb_t)um_umem_buffer_cb, ump, addr) ==
4047 -1) {
4048 mdb_warn("can't walk 'umem' for cache %p", addr);
4049 return (WALK_ERR);
4050 }
4051
4052 return (WALK_NEXT);
4053 }
4054
4055 void
4056 umem_malloc_dist_help(void)
4057 {
4058 mdb_printf("%s\n",
4059 "report distribution of outstanding malloc()s");
4060 mdb_dec_indent(2);
4061 mdb_printf("%<b>OPTIONS%</b>\n");
4062 mdb_inc_indent(2);
4063 mdb_printf("%s",
4064 " -b maxbins\n"
4065 " Use at most maxbins bins for the data\n"
4066 " -B minbinsize\n"
4067 " Make the bins at least minbinsize bytes apart\n"
4068 " -d dump the raw data out, without binning\n"
4069 " -g use geometric binning instead of linear binning\n");
4070 }
4071
4072 /*ARGSUSED*/
4073 int
4074 umem_malloc_dist(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
4075 {
4076 umem_malloc_info_t mi;
4077 uint_t geometric = 0;
4078 uint_t dump = 0;
4079 size_t maxbuckets = 0;
4080 size_t minbucketsize = 0;
4081
4082 size_t minalloc = 0;
4083 size_t maxalloc = UMI_MAX_BUCKET;
4084
4085 if (flags & DCMD_ADDRSPEC)
4086 return (DCMD_USAGE);
4087
4088 if (mdb_getopts(argc, argv,
4089 'd', MDB_OPT_SETBITS, TRUE, &dump,
4090 'g', MDB_OPT_SETBITS, TRUE, &geometric,
4091 'b', MDB_OPT_UINTPTR, &maxbuckets,
4092 'B', MDB_OPT_UINTPTR, &minbucketsize,
4093 0) != argc)
4094 return (DCMD_USAGE);
4095
4096 bzero(&mi, sizeof (mi));
4097 mi.um_bucket = mdb_zalloc((UMI_MAX_BUCKET + 1) * sizeof (*mi.um_bucket),
4098 UM_SLEEP | UM_GC);
4099
4100 if (mdb_walk("umem_cache", (mdb_walk_cb_t)um_umem_cache_cb,
4101 &mi) == -1) {
4102 mdb_warn("unable to walk 'umem_cache'");
4103 return (DCMD_ERR);
4104 }
4105
4106 if (dump) {
4107 int i;
4108 for (i = minalloc; i <= maxalloc; i++)
4109 mdb_printf("%d\t%d\n", i, mi.um_bucket[i]);
4110
4111 return (DCMD_OK);
4112 }
4113
4114 umem_malloc_print_dist(mi.um_bucket, minalloc, maxalloc,
4115 maxbuckets, minbucketsize, geometric);
4116
4117 return (DCMD_OK);
4118 }
4119
4120 void
4121 umem_malloc_info_help(void)
4122 {
4123 mdb_printf("%s\n",
4124 "report information about malloc()s by cache. ");
4125 mdb_dec_indent(2);
4126 mdb_printf("%<b>OPTIONS%</b>\n");
4127 mdb_inc_indent(2);
4128 mdb_printf("%s",
4129 " -b maxbins\n"
4130 " Use at most maxbins bins for the data\n"
4131 " -B minbinsize\n"
4132 " Make the bins at least minbinsize bytes apart\n"
4133 " -d dump the raw distribution data without binning\n"
4134 #ifndef _KMDB
4135 " -g use geometric binning instead of linear binning\n"
4136 #endif
4137 "");
4138 }
4139 int
4140 umem_malloc_info(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
4141 {
4142 umem_cache_t c;
4143 umem_malloc_info_t mi;
4144
4145 int skip = 0;
4146
4147 size_t maxmalloc;
4148 size_t overhead;
4149 size_t allocated;
4150 size_t avg_malloc;
4151 size_t overhead_pct; /* 1000 * overhead_percent */
4152
4153 uint_t verbose = 0;
4154 uint_t dump = 0;
4155 uint_t geometric = 0;
4156 size_t maxbuckets = 0;
4157 size_t minbucketsize = 0;
4158
4159 int *alloc_sizes;
4160 int idx;
4161 size_t num;
4162 size_t minmalloc;
4163
4164 if (mdb_getopts(argc, argv,
4165 'd', MDB_OPT_SETBITS, TRUE, &dump,
4166 'g', MDB_OPT_SETBITS, TRUE, &geometric,
4167 'b', MDB_OPT_UINTPTR, &maxbuckets,
4168 'B', MDB_OPT_UINTPTR, &minbucketsize,
4169 0) != argc)
4170 return (DCMD_USAGE);
4171
4172 if (dump || geometric || (maxbuckets != 0) || (minbucketsize != 0))
4173 verbose = 1;
4174
4175 if (!(flags & DCMD_ADDRSPEC)) {
4176 if (mdb_walk_dcmd("umem_cache", "umem_malloc_info",
4177 argc, argv) == -1) {
4178 mdb_warn("can't walk umem_cache");
4179 return (DCMD_ERR);
4180 }
4181 return (DCMD_OK);
4182 }
4183
4184 if (!mdb_vread(&c, sizeof (c), addr)) {
4185 mdb_warn("unable to read cache at %p", addr);
4186 return (DCMD_ERR);
4187 }
4188
4189 if (strncmp(c.cache_name, "umem_alloc_", strlen("umem_alloc_")) != 0) {
4190 if (!(flags & DCMD_LOOP))
4191 mdb_warn("umem_malloc_info: cache \"%s\" is not used "
4192 "by malloc()\n", c.cache_name);
4193 skip = 1;
4194 }
4195
4196 /*
4197 * normally, print the header only the first time. In verbose mode,
4198 * print the header on every non-skipped buffer
4199 */
4200 if ((!verbose && DCMD_HDRSPEC(flags)) || (verbose && !skip))
4201 mdb_printf("%<ul>%-?s %6s %6s %8s %8s %10s %10s %6s%</ul>\n",
4202 "CACHE", "BUFSZ", "MAXMAL",
4203 "BUFMALLC", "AVG_MAL", "MALLOCED", "OVERHEAD", "%OVER");
4204
4205 if (skip)
4206 return (DCMD_OK);
4207
4208 maxmalloc = c.cache_bufsize - sizeof (struct malloc_data);
4209 #ifdef _LP64
4210 if (c.cache_bufsize > UMEM_SECOND_ALIGN)
4211 maxmalloc -= sizeof (struct malloc_data);
4212 #endif
4213
4214 bzero(&mi, sizeof (mi));
4215 mi.um_cp = &c;
4216 if (verbose)
4217 mi.um_bucket =
4218 mdb_zalloc((UMI_MAX_BUCKET + 1) * sizeof (*mi.um_bucket),
4219 UM_SLEEP | UM_GC);
4220
4221 if (mdb_pwalk("umem", (mdb_walk_cb_t)um_umem_buffer_cb, &mi, addr) ==
4222 -1) {
4223 mdb_warn("can't walk 'umem'");
4224 return (DCMD_ERR);
4225 }
4226
4227 overhead = mi.um_malloc_overhead;
4228 allocated = mi.um_malloc_size;
4229
4230 /* do integer round off for the average */
4231 if (mi.um_malloc != 0)
4232 avg_malloc = (allocated + (mi.um_malloc - 1)/2) / mi.um_malloc;
4233 else
4234 avg_malloc = 0;
4235
4236 /*
4237 * include per-slab overhead
4238 *
4239 * Each slab in a given cache is the same size, and has the same
4240 * number of chunks in it; we read in the first slab on the
4241 * slab list to get the number of chunks for all slabs. To
4242 * compute the per-slab overhead, we just subtract the chunk usage
4243 * from the slabsize:
4244 *
4245 * +------------+-------+-------+ ... --+-------+-------+-------+
4246 * |////////////| | | ... | |///////|///////|
4247 * |////color///| chunk | chunk | ... | chunk |/color/|/slab//|
4248 * |////////////| | | ... | |///////|///////|
4249 * +------------+-------+-------+ ... --+-------+-------+-------+
4250 * | \_______chunksize * chunks_____/ |
4251 * \__________________________slabsize__________________________/
4252 *
4253 * For UMF_HASH caches, there is an additional source of overhead;
4254 * the external umem_slab_t and per-chunk bufctl structures. We
4255 * include those in our per-slab overhead.
4256 *
4257 * Once we have a number for the per-slab overhead, we estimate
4258 * the actual overhead by treating the malloc()ed buffers as if
4259 * they were densely packed:
4260 *
4261 * additional overhead = (# mallocs) * (per-slab) / (chunks);
4262 *
4263 * carefully ordering the multiply before the divide, to avoid
4264 * round-off error.
4265 */
4266 if (mi.um_malloc != 0) {
4267 umem_slab_t slab;
4268 uintptr_t saddr = (uintptr_t)c.cache_nullslab.slab_next;
4269
4270 if (mdb_vread(&slab, sizeof (slab), saddr) == -1) {
4271 mdb_warn("unable to read slab at %p\n", saddr);
4272 } else {
4273 long chunks = slab.slab_chunks;
4274 if (chunks != 0 && c.cache_chunksize != 0 &&
4275 chunks <= c.cache_slabsize / c.cache_chunksize) {
4276 uintmax_t perslab =
4277 c.cache_slabsize -
4278 (c.cache_chunksize * chunks);
4279
4280 if (c.cache_flags & UMF_HASH) {
4281 perslab += sizeof (umem_slab_t) +
4282 chunks *
4283 ((c.cache_flags & UMF_AUDIT) ?
4284 sizeof (umem_bufctl_audit_t) :
4285 sizeof (umem_bufctl_t));
4286 }
4287 overhead +=
4288 (perslab * (uintmax_t)mi.um_malloc)/chunks;
4289 } else {
4290 mdb_warn("invalid #chunks (%d) in slab %p\n",
4291 chunks, saddr);
4292 }
4293 }
4294 }
4295
4296 if (allocated != 0)
4297 overhead_pct = (1000ULL * overhead) / allocated;
4298 else
4299 overhead_pct = 0;
4300
4301 mdb_printf("%0?p %6ld %6ld %8ld %8ld %10ld %10ld %3ld.%01ld%%\n",
4302 addr, c.cache_bufsize, maxmalloc,
4303 mi.um_malloc, avg_malloc, allocated, overhead,
4304 overhead_pct / 10, overhead_pct % 10);
4305
4306 if (!verbose)
4307 return (DCMD_OK);
4308
4309 if (!dump)
4310 mdb_printf("\n");
4311
4312 if (get_umem_alloc_sizes(&alloc_sizes, &num) == -1)
4313 return (DCMD_ERR);
4314
4315 for (idx = 0; idx < num; idx++) {
4316 if (alloc_sizes[idx] == c.cache_bufsize)
4317 break;
4318 if (alloc_sizes[idx] == 0) {
4319 idx = num; /* 0-terminated array */
4320 break;
4321 }
4322 }
4323 if (idx == num) {
4324 mdb_warn(
4325 "cache %p's size (%d) not in umem_alloc_sizes\n",
4326 addr, c.cache_bufsize);
4327 return (DCMD_ERR);
4328 }
4329
4330 minmalloc = (idx == 0)? 0 : alloc_sizes[idx - 1];
4331 if (minmalloc > 0) {
4332 #ifdef _LP64
4333 if (minmalloc > UMEM_SECOND_ALIGN)
4334 minmalloc -= sizeof (struct malloc_data);
4335 #endif
4336 minmalloc -= sizeof (struct malloc_data);
4337 minmalloc += 1;
4338 }
4339
4340 if (dump) {
4341 for (idx = minmalloc; idx <= maxmalloc; idx++)
4342 mdb_printf("%d\t%d\n", idx, mi.um_bucket[idx]);
4343 mdb_printf("\n");
4344 } else {
4345 umem_malloc_print_dist(mi.um_bucket, minmalloc, maxmalloc,
4346 maxbuckets, minbucketsize, geometric);
4347 }
4348
4349 return (DCMD_OK);
4350 }