Print this page
10908 Simplify SMAP relocations with krtld
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/krtld/kobj.c
+++ new/usr/src/uts/common/krtld/kobj.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25 /*
26 26 * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
27 27 * All rights reserved. Use is subject to license terms.
28 28 * Copyright (c) 2018, Joyent, Inc.
29 29 */
30 30
31 31 /*
32 32 * Kernel's linker/loader
33 33 */
34 34
35 35 #include <sys/types.h>
36 36 #include <sys/param.h>
37 37 #include <sys/sysmacros.h>
38 38 #include <sys/systm.h>
39 39 #include <sys/user.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/reboot.h>
42 42 #include <sys/bootconf.h>
43 43 #include <sys/debug.h>
44 44 #include <sys/uio.h>
45 45 #include <sys/file.h>
46 46 #include <sys/vnode.h>
47 47 #include <sys/user.h>
48 48 #include <sys/mman.h>
49 49 #include <vm/as.h>
50 50 #include <vm/seg_kp.h>
51 51 #include <vm/seg_kmem.h>
52 52 #include <sys/elf.h>
53 53 #include <sys/elf_notes.h>
54 54 #include <sys/vmsystm.h>
55 55 #include <sys/kdi.h>
56 56 #include <sys/atomic.h>
57 57 #include <sys/kmdb.h>
58 58
59 59 #include <sys/link.h>
60 60 #include <sys/kobj.h>
61 61 #include <sys/ksyms.h>
62 62 #include <sys/disp.h>
63 63 #include <sys/modctl.h>
64 64 #include <sys/varargs.h>
65 65 #include <sys/kstat.h>
66 66 #include <sys/kobj_impl.h>
67 67 #include <sys/fs/decomp.h>
68 68 #include <sys/callb.h>
69 69 #include <sys/cmn_err.h>
70 70 #include <sys/tnf_probe.h>
71 71 #include <sys/zmod.h>
72 72
73 73 #include <krtld/reloc.h>
74 74 #include <krtld/kobj_kdi.h>
75 75 #include <sys/sha1.h>
76 76 #include <sys/crypto/elfsign.h>
77 77
78 78 #if !defined(_OBP)
79 79 #include <sys/bootvfs.h>
80 80 #endif
81 81
82 82 /*
83 83 * do_symbols() error codes
84 84 */
85 85 #define DOSYM_UNDEF -1 /* undefined symbol */
86 86 #define DOSYM_UNSAFE -2 /* MT-unsafe driver symbol */
87 87
88 88 #if !defined(_OBP)
89 89 static void synthetic_bootaux(char *, val_t *);
90 90 #endif
91 91
92 92 static struct module *load_exec(val_t *, char *);
93 93 static void load_linker(val_t *);
94 94 static struct modctl *add_primary(const char *filename, int);
95 95 static int bind_primary(val_t *, int);
96 96 static int load_primary(struct module *, int);
97 97 static int load_kmdb(val_t *);
98 98 static int get_progbits(struct module *, struct _buf *);
99 99 static int get_syms(struct module *, struct _buf *);
100 100 static int get_ctf(struct module *, struct _buf *);
101 101 static void get_signature(struct module *, struct _buf *);
102 102 static int do_common(struct module *);
103 103 static void add_dependent(struct module *, struct module *);
104 104 static int do_dependents(struct modctl *, char *, size_t);
105 105 static int do_symbols(struct module *, Elf64_Addr);
106 106 static void module_assign(struct modctl *, struct module *);
107 107 static void free_module_data(struct module *);
108 108 static char *depends_on(struct module *);
109 109 static char *getmodpath(const char *);
110 110 static char *basename(char *);
111 111 static void attr_val(val_t *);
112 112 static char *find_libmacro(char *);
113 113 static char *expand_libmacro(char *, char *, char *);
114 114 static int read_bootflags(void);
115 115 static int kobj_comp_setup(struct _buf *, struct compinfo *);
116 116 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
117 117 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
118 118 static int kobj_boot_open(char *, int);
119 119 static int kobj_boot_close(int);
120 120 static int kobj_boot_seek(int, off_t, off_t);
121 121 static int kobj_boot_read(int, caddr_t, size_t);
122 122 static int kobj_boot_fstat(int, struct bootstat *);
123 123 static int kobj_boot_compinfo(int, struct compinfo *);
124 124
125 125 static Sym *lookup_one(struct module *, const char *);
126 126 static void sym_insert(struct module *, char *, symid_t);
127 127 static Sym *sym_lookup(struct module *, Sym *);
128 128
129 129 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
130 130 static void kobjopen_free(struct kobjopen_tctl *ltp);
131 131 static void kobjopen_thread(struct kobjopen_tctl *ltp);
132 132 static int kobj_is_compressed(intptr_t);
133 133
134 134 extern int kcopy(const void *, void *, size_t);
135 135 extern int elf_mach_ok(Ehdr *);
136 136 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
137 137
138 138 #if !defined(_OBP)
139 139 extern int kobj_boot_mountroot(void);
140 140 #endif
141 141
142 142 static void tnf_unsplice_probes(uint_t, struct modctl *);
143 143 extern tnf_probe_control_t *__tnf_probe_list_head;
144 144 extern tnf_tag_data_t *__tnf_tag_list_head;
145 145
146 146 extern int modrootloaded;
147 147 extern int swaploaded;
148 148 extern int bop_io_quiesced;
149 149 extern int last_module_id;
150 150
151 151 extern char stubs_base[];
152 152 extern char stubs_end[];
153 153
154 154 #ifdef KOBJ_DEBUG
155 155 /*
156 156 * Values that can be or'd in to kobj_debug and their effects:
157 157 *
158 158 * D_DEBUG - misc. debugging information.
159 159 * D_SYMBOLS - list symbols and their values as they are entered
160 160 * into the hash table
161 161 * D_RELOCATIONS - display relocation processing information
162 162 * D_LOADING - display information about each module as it
163 163 * is loaded.
164 164 */
165 165 int kobj_debug = 0;
166 166
167 167 #define KOBJ_MARK(s) if (kobj_debug & D_DEBUG) \
168 168 (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
169 169 #else
170 170 #define KOBJ_MARK(s) /* discard */
171 171 #endif
172 172
173 173 #define MODPATH_PROPNAME "module-path"
174 174
175 175 #ifdef MODDIR_SUFFIX
176 176 static char slash_moddir_suffix_slash[] = MODDIR_SUFFIX "/";
177 177 #else
178 178 #define slash_moddir_suffix_slash ""
179 179 #endif
180 180
181 181 #define _moddebug get_weakish_int(&moddebug)
182 182 #define _modrootloaded get_weakish_int(&modrootloaded)
183 183 #define _swaploaded get_weakish_int(&swaploaded)
184 184 #define _ioquiesced get_weakish_int(&bop_io_quiesced)
185 185
186 186 #define mod(X) (struct module *)((X)->modl_modp->mod_mp)
187 187
188 188 void *romp; /* rom vector (opaque to us) */
189 189 struct bootops *ops; /* bootops vector */
190 190 void *dbvec; /* debug vector */
191 191
192 192 /*
193 193 * kobjopen thread control structure
194 194 */
195 195 struct kobjopen_tctl {
196 196 ksema_t sema;
197 197 char *name; /* name of file */
198 198 struct vnode *vp; /* vnode return from vn_open() */
199 199 int Errno; /* error return from vnopen */
200 200 };
201 201
202 202 /*
203 203 * Structure for defining dynamically expandable library macros
204 204 */
205 205
206 206 struct lib_macro_info {
207 207 char *lmi_list; /* ptr to list of possible choices */
208 208 char *lmi_macroname; /* pointer to macro name */
209 209 ushort_t lmi_ba_index; /* index into bootaux vector */
210 210 ushort_t lmi_macrolen; /* macro length */
211 211 } libmacros[] = {
212 212 { NULL, "CPU", BA_CPU, 0 },
213 213 { NULL, "MMU", BA_MMU, 0 }
214 214 };
215 215
216 216 #define NLIBMACROS sizeof (libmacros) / sizeof (struct lib_macro_info)
217 217
218 218 char *boot_cpu_compatible_list; /* make $CPU available */
219 219
220 220 char *kobj_module_path; /* module search path */
221 221 vmem_t *text_arena; /* module text arena */
222 222 static vmem_t *data_arena; /* module data & bss arena */
223 223 static vmem_t *ctf_arena; /* CTF debug data arena */
224 224 static struct modctl *kobj_modules = NULL; /* modules loaded */
225 225 int kobj_mmu_pagesize; /* system pagesize */
226 226 static int lg_pagesize; /* "large" pagesize */
227 227 static int kobj_last_module_id = 0; /* id assignment */
228 228 static kmutex_t kobj_lock; /* protects mach memory list */
229 229
230 230 /*
231 231 * The following functions have been implemented by the kernel.
232 232 * However, many 3rd party drivers provide their own implementations
233 233 * of these functions. When such drivers are loaded, messages
234 234 * indicating that these symbols have been multiply defined will be
235 235 * emitted to the console. To avoid alarming customers for no good
236 236 * reason, we simply suppress such warnings for the following set of
237 237 * functions.
238 238 */
239 239 static char *suppress_sym_list[] =
240 240 {
241 241 "strstr",
242 242 "strncat",
243 243 "strlcat",
244 244 "strlcpy",
245 245 "strspn",
246 246 "memcpy",
247 247 "memset",
248 248 "memmove",
249 249 "memcmp",
250 250 "memchr",
251 251 "__udivdi3",
252 252 "__divdi3",
253 253 "__umoddi3",
254 254 "__moddi3",
255 255 NULL /* This entry must exist */
256 256 };
257 257
258 258 /* indexed by KOBJ_NOTIFY_* */
259 259 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
260 260
261 261 /*
262 262 * TNF probe management globals
263 263 */
264 264 tnf_probe_control_t *__tnf_probe_list_head = NULL;
265 265 tnf_tag_data_t *__tnf_tag_list_head = NULL;
266 266 int tnf_changed_probe_list = 0;
267 267
268 268 /*
269 269 * Prefix for statically defined tracing (SDT) DTrace probes.
270 270 */
271 271 const char *sdt_prefix = "__dtrace_probe_";
272 272
273 273 /*
274 274 * Beginning and end of the kernel's dynamic text/data segments.
275 275 */
276 276 static caddr_t _text;
277 277 static caddr_t _etext;
278 278 static caddr_t _data;
279 279
280 280 /*
281 281 * The sparc linker doesn't create a memory location
282 282 * for a variable named _edata, so _edata can only be
283 283 * referred to, not modified. krtld needs a static
284 284 * variable to modify it - within krtld, of course -
285 285 * outside of krtld, e_data is used in all kernels.
286 286 */
287 287 #if defined(__sparc)
288 288 static caddr_t _edata;
289 289 #else
290 290 extern caddr_t _edata;
291 291 #endif
292 292
293 293 Addr dynseg = 0; /* load address of "dynamic" segment */
294 294 size_t dynsize; /* "dynamic" segment size */
295 295
296 296
297 297 int standalone = 1; /* an unwholey kernel? */
298 298 int use_iflush; /* iflush after relocations */
299 299
300 300 /*
301 301 * _kobj_printf() and _vkobj_printf()
302 302 *
303 303 * Common printf function pointer. Can handle only one conversion
304 304 * specification in the format string. Some of the functions invoked
305 305 * through this function pointer cannot handle more that one conversion
306 306 * specification in the format string.
307 307 */
308 308 void (*_kobj_printf)(void *, const char *, ...); /* printf routine */
309 309 void (*_vkobj_printf)(void *, const char *, va_list); /* vprintf routine */
310 310
311 311 /*
312 312 * Standalone function pointers for use within krtld.
313 313 * Many platforms implement optimized platmod versions of
314 314 * utilities such as bcopy and any such are not yet available
315 315 * until the kernel is more completely stitched together.
316 316 * See kobj_impl.h
317 317 */
318 318 void (*kobj_bcopy)(const void *, void *, size_t);
319 319 void (*kobj_bzero)(void *, size_t);
320 320 size_t (*kobj_strlcat)(char *, const char *, size_t);
321 321
322 322 static kobj_stat_t kobj_stat;
323 323
324 324 #define MINALIGN 8 /* at least a double-word */
325 325
326 326 int
327 327 get_weakish_int(int *ip)
328 328 {
329 329 if (standalone)
330 330 return (0);
331 331 return (ip == NULL ? 0 : *ip);
332 332 }
333 333
334 334 static void *
335 335 get_weakish_pointer(void **ptrp)
336 336 {
337 337 if (standalone)
338 338 return (0);
339 339 return (ptrp == NULL ? 0 : *ptrp);
340 340 }
341 341
342 342 /*
343 343 * XXX fix dependencies on "kernel"; this should work
344 344 * for other standalone binaries as well.
345 345 *
346 346 * XXX Fix hashing code to use one pointer to
347 347 * hash entries.
348 348 * |----------|
349 349 * | nbuckets |
350 350 * |----------|
351 351 * | nchains |
352 352 * |----------|
353 353 * | bucket[] |
354 354 * |----------|
355 355 * | chain[] |
356 356 * |----------|
357 357 */
358 358
359 359 /*
360 360 * Load, bind and relocate all modules that
361 361 * form the primary kernel. At this point, our
362 362 * externals have not been relocated.
363 363 */
364 364 void
365 365 kobj_init(
366 366 void *romvec,
367 367 void *dvec,
368 368 struct bootops *bootvec,
369 369 val_t *bootaux)
370 370 {
371 371 struct module *mp;
372 372 struct modctl *modp;
373 373 Addr entry;
374 374 char filename[MAXPATHLEN];
375 375
376 376 /*
377 377 * Save these to pass on to
378 378 * the booted standalone.
379 379 */
380 380 romp = romvec;
381 381 dbvec = dvec;
382 382
383 383 ops = bootvec;
384 384 kobj_setup_standalone_vectors();
385 385
386 386 KOBJ_MARK("Entered kobj_init()");
387 387
388 388 (void) BOP_GETPROP(ops, "whoami", filename);
389 389
390 390 /*
391 391 * We don't support standalone debuggers anymore. The use of kadb
392 392 * will interfere with the later use of kmdb. Let the user mend
393 393 * their ways now. Users will reach this message if they still
394 394 * have the kadb binary on their system (perhaps they used an old
395 395 * bfu, or maybe they intentionally copied it there) and have
396 396 * specified its use in a way that eluded our checking in the boot
397 397 * program.
398 398 */
399 399 if (dvec != NULL) {
400 400 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
401 401 "kadb are no longer supported\n\n");
402 402 goto fail;
403 403 }
404 404
405 405 #if defined(_OBP)
406 406 /*
407 407 * OBP allows us to read both the ramdisk and
408 408 * the underlying root fs when root is a disk.
409 409 * This can lower incidences of unbootable systems
410 410 * when the archive is out-of-date with the /etc
411 411 * state files.
412 412 */
413 413 if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
414 414 _kobj_printf(ops, "can't mount boot fs\n");
415 415 goto fail;
416 416 }
417 417 #else
418 418 {
419 419 /* on x86, we always boot with a ramdisk */
420 420 (void) kobj_boot_mountroot();
421 421
422 422 /*
423 423 * Now that the ramdisk is mounted, finish boot property
424 424 * initialization.
425 425 */
426 426 boot_prop_finish();
427 427 }
428 428
429 429 #if !defined(_UNIX_KRTLD)
430 430 /*
431 431 * 'unix' is linked together with 'krtld' into one executable and
432 432 * the early boot code does -not- hand us any of the dynamic metadata
433 433 * about the executable. In particular, it does not read in, map or
434 434 * otherwise look at the program headers. We fake all that up now.
435 435 *
436 436 * We do this early as DTrace static probes and tnf probes both call
437 437 * undefined references. We have to process those relocations before
438 438 * calling any of them.
439 439 *
440 440 * OBP tells kobj_start() where the ELF image is in memory, so it
441 441 * synthesized bootaux before kobj_init() was called
442 442 */
443 443 if (bootaux[BA_PHDR].ba_ptr == NULL)
444 444 synthetic_bootaux(filename, bootaux);
445 445
446 446 #endif /* !_UNIX_KRTLD */
447 447 #endif /* _OBP */
448 448
449 449 /*
450 450 * Save the interesting attribute-values
451 451 * (scanned by kobj_boot).
452 452 */
453 453 attr_val(bootaux);
454 454
455 455 /*
456 456 * Set the module search path.
457 457 */
458 458 kobj_module_path = getmodpath(filename);
459 459
460 460 boot_cpu_compatible_list = find_libmacro("CPU");
461 461
462 462 /*
463 463 * These two modules have actually been
464 464 * loaded by boot, but we finish the job
465 465 * by introducing them into the world of
466 466 * loadable modules.
467 467 */
468 468
469 469 mp = load_exec(bootaux, filename);
470 470 load_linker(bootaux);
471 471
472 472 /*
473 473 * Load all the primary dependent modules.
474 474 */
475 475 if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
476 476 goto fail;
477 477
478 478 /*
479 479 * Glue it together.
480 480 */
481 481 if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
482 482 goto fail;
483 483
484 484 entry = bootaux[BA_ENTRY].ba_val;
485 485
486 486 /*
487 487 * Get the boot flags
488 488 */
489 489 bootflags(ops);
490 490
491 491 if (boothowto & RB_VERBOSE)
492 492 kobj_lm_dump(KOBJ_LM_PRIMARY);
493 493
494 494 kobj_kdi_init();
495 495
496 496 if (boothowto & RB_KMDB) {
497 497 if (load_kmdb(bootaux) < 0)
498 498 goto fail;
499 499 }
500 500
501 501 /*
502 502 * Post setup.
503 503 */
504 504 s_text = _text;
505 505 e_text = _etext;
506 506 s_data = _data;
507 507 e_data = _edata;
508 508
509 509 kobj_sync_instruction_memory(s_text, e_text - s_text);
510 510
511 511 #ifdef KOBJ_DEBUG
512 512 if (kobj_debug & D_DEBUG)
513 513 _kobj_printf(ops,
514 514 "krtld: transferring control to: 0x%p\n", entry);
515 515 #endif
516 516
517 517 /*
518 518 * Make sure the mod system knows about the modules already loaded.
519 519 */
520 520 last_module_id = kobj_last_module_id;
521 521 bcopy(kobj_modules, &modules, sizeof (modules));
522 522 modp = &modules;
523 523 do {
524 524 if (modp->mod_next == kobj_modules)
525 525 modp->mod_next = &modules;
526 526 if (modp->mod_prev == kobj_modules)
527 527 modp->mod_prev = &modules;
528 528 } while ((modp = modp->mod_next) != &modules);
529 529
530 530 standalone = 0;
531 531
532 532 #ifdef KOBJ_DEBUG
533 533 if (kobj_debug & D_DEBUG)
534 534 _kobj_printf(ops,
535 535 "krtld: really transferring control to: 0x%p\n", entry);
536 536 #endif
537 537
538 538 /* restore printf/bcopy/bzero vectors before returning */
539 539 kobj_restore_vectors();
540 540
541 541 #if defined(_DBOOT)
542 542 /*
543 543 * krtld was called from a dboot ELF section, the embedded
544 544 * dboot code contains the real entry via bootaux
545 545 */
546 546 exitto((caddr_t)entry);
547 547 #else
548 548 /*
549 549 * krtld was directly called from startup
550 550 */
551 551 return;
552 552 #endif
553 553
554 554 fail:
555 555
556 556 _kobj_printf(ops, "krtld: error during initial load/link phase\n");
557 557
558 558 #if !defined(_UNIX_KRTLD)
559 559 _kobj_printf(ops, "\n");
560 560 _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
561 561 " for:\n");
562 562 _kobj_printf(ops, " %s\n", filename);
563 563 _kobj_printf(ops, "in the boot archive. Please verify that this"
564 564 " file\n");
565 565 _kobj_printf(ops, "matches what is found in the boot archive.\n");
566 566 _kobj_printf(ops, "You may need to boot using the Solaris failsafe to"
567 567 " fix this.\n");
568 568 bop_panic("Unable to boot");
569 569 #endif
570 570 }
571 571
572 572 #if !defined(_UNIX_KRTLD) && !defined(_OBP)
573 573 /*
574 574 * Synthesize additional metadata that describes the executable if
575 575 * krtld's caller didn't do it.
576 576 *
577 577 * (When the dynamic executable has an interpreter, the boot program
578 578 * does all this for us. Where we don't have an interpreter, (or a
579 579 * even a boot program, perhaps) we have to do this for ourselves.)
580 580 */
581 581 static void
582 582 synthetic_bootaux(char *filename, val_t *bootaux)
583 583 {
584 584 Ehdr ehdr;
585 585 caddr_t phdrbase;
586 586 struct _buf *file;
587 587 int i, n;
588 588
589 589 /*
590 590 * Elf header
591 591 */
592 592 KOBJ_MARK("synthetic_bootaux()");
593 593 KOBJ_MARK(filename);
594 594 file = kobj_open_file(filename);
595 595 if (file == (struct _buf *)-1) {
596 596 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
597 597 return;
598 598 }
599 599 KOBJ_MARK("reading program headers");
600 600 if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
601 601 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
602 602 filename);
603 603 return;
604 604 }
605 605
606 606 /*
607 607 * Program headers
608 608 */
609 609 bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
610 610 bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
611 611 n = ehdr.e_phentsize * ehdr.e_phnum;
612 612
613 613 phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
614 614
615 615 if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
616 616 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
617 617 filename);
618 618 return;
619 619 }
620 620 bootaux[BA_PHDR].ba_ptr = phdrbase;
621 621 kobj_close_file(file);
622 622 KOBJ_MARK("closed file");
623 623
624 624 /*
625 625 * Find the dynamic section address
626 626 */
627 627 for (i = 0; i < ehdr.e_phnum; i++) {
628 628 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
629 629
630 630 if (phdr->p_type == PT_DYNAMIC) {
631 631 bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
632 632 break;
633 633 }
634 634 }
635 635 KOBJ_MARK("synthetic_bootaux() done");
636 636 }
637 637 #endif /* !_UNIX_KRTLD && !_OBP */
638 638
639 639 /*
640 640 * Set up any global information derived
641 641 * from attribute/values in the boot or
642 642 * aux vector.
643 643 */
644 644 static void
645 645 attr_val(val_t *bootaux)
646 646 {
647 647 Phdr *phdr;
648 648 int phnum, phsize;
649 649 int i;
650 650
651 651 KOBJ_MARK("attr_val()");
652 652 kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
653 653 lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
654 654 use_iflush = bootaux[BA_IFLUSH].ba_val;
655 655
656 656 phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
657 657 phnum = bootaux[BA_PHNUM].ba_val;
658 658 phsize = bootaux[BA_PHENT].ba_val;
659 659 for (i = 0; i < phnum; i++) {
660 660 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
661 661
662 662 if (phdr->p_type != PT_LOAD) {
663 663 continue;
664 664 }
665 665 /*
666 666 * Bounds of the various segments.
667 667 */
668 668 if (!(phdr->p_flags & PF_X)) {
669 669 #if defined(_RELSEG)
670 670 /*
671 671 * sparc kernel puts the dynamic info
672 672 * into a separate segment, which is
673 673 * free'd in bop_fini()
674 674 */
675 675 ASSERT(phdr->p_vaddr != 0);
676 676 dynseg = phdr->p_vaddr;
677 677 dynsize = phdr->p_memsz;
678 678 #else
679 679 ASSERT(phdr->p_vaddr == 0);
680 680 #endif
681 681 } else {
682 682 if (phdr->p_flags & PF_W) {
683 683 _data = (caddr_t)phdr->p_vaddr;
684 684 _edata = _data + phdr->p_memsz;
685 685 } else {
686 686 _text = (caddr_t)phdr->p_vaddr;
687 687 _etext = _text + phdr->p_memsz;
688 688 }
689 689 }
690 690 }
691 691
692 692 /* To do the kobj_alloc, _edata needs to be set. */
693 693 for (i = 0; i < NLIBMACROS; i++) {
694 694 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
695 695 libmacros[i].lmi_list = kobj_alloc(
696 696 strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
697 697 1, KM_WAIT);
698 698 (void) strcpy(libmacros[i].lmi_list,
699 699 bootaux[libmacros[i].lmi_ba_index].ba_ptr);
700 700 }
701 701 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
702 702 }
703 703 }
704 704
705 705 /*
706 706 * Set up the booted executable.
707 707 */
708 708 static struct module *
709 709 load_exec(val_t *bootaux, char *filename)
710 710 {
711 711 struct modctl *cp;
712 712 struct module *mp;
713 713 Dyn *dyn;
714 714 Sym *sp;
715 715 int i, lsize, osize, nsize, allocsize;
716 716 char *libname, *tmp;
717 717 char path[MAXPATHLEN];
718 718
719 719 #ifdef KOBJ_DEBUG
720 720 if (kobj_debug & D_DEBUG)
721 721 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
722 722 #endif
723 723
724 724 KOBJ_MARK("add_primary");
725 725 cp = add_primary(filename, KOBJ_LM_PRIMARY);
726 726
727 727 KOBJ_MARK("struct module");
728 728 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
729 729 cp->mod_mp = mp;
730 730
731 731 /*
732 732 * We don't have the following information
733 733 * since this module is an executable and not
734 734 * a relocatable .o.
735 735 */
736 736 mp->symtbl_section = 0;
737 737 mp->shdrs = NULL;
738 738 mp->strhdr = NULL;
739 739
740 740 /*
741 741 * Since this module is the only exception,
742 742 * we cons up some section headers.
743 743 */
744 744 KOBJ_MARK("symhdr");
745 745 mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
746 746
747 747 KOBJ_MARK("strhdr");
748 748 mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
749 749
750 750 mp->symhdr->sh_type = SHT_SYMTAB;
751 751 mp->strhdr->sh_type = SHT_STRTAB;
752 752 /*
753 753 * Scan the dynamic structure.
754 754 */
755 755 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
756 756 dyn->d_tag != DT_NULL; dyn++) {
757 757 switch (dyn->d_tag) {
758 758 case DT_SYMTAB:
759 759 mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
760 760 mp->symhdr->sh_addr = dyn->d_un.d_ptr;
761 761 break;
762 762 case DT_HASH:
763 763 mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
764 764 mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
765 765 break;
766 766 case DT_STRTAB:
767 767 mp->strings = (char *)dyn->d_un.d_ptr;
768 768 mp->strhdr->sh_addr = dyn->d_un.d_ptr;
769 769 break;
770 770 case DT_STRSZ:
771 771 mp->strhdr->sh_size = dyn->d_un.d_val;
772 772 break;
773 773 case DT_SYMENT:
774 774 mp->symhdr->sh_entsize = dyn->d_un.d_val;
775 775 break;
776 776 }
777 777 }
778 778
779 779 /*
780 780 * Collapse any DT_NEEDED entries into one string.
781 781 */
782 782 nsize = osize = 0;
783 783 allocsize = MAXPATHLEN;
784 784
785 785 KOBJ_MARK("depends_on");
786 786 mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
787 787
788 788 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
789 789 dyn->d_tag != DT_NULL; dyn++)
790 790 if (dyn->d_tag == DT_NEEDED) {
791 791 char *_lib;
792 792
793 793 libname = mp->strings + dyn->d_un.d_val;
794 794 if (strchr(libname, '$') != NULL) {
795 795 if ((_lib = expand_libmacro(libname,
796 796 path, path)) != NULL)
797 797 libname = _lib;
798 798 else
799 799 _kobj_printf(ops, "krtld: "
800 800 "load_exec: fail to "
801 801 "expand %s\n", libname);
802 802 }
803 803 lsize = strlen(libname);
804 804 nsize += lsize;
805 805 if (nsize + 1 > allocsize) {
806 806 KOBJ_MARK("grow depends_on");
807 807 tmp = kobj_alloc(allocsize + MAXPATHLEN,
808 808 KM_WAIT);
809 809 bcopy(mp->depends_on, tmp, osize);
810 810 kobj_free(mp->depends_on, allocsize);
811 811 mp->depends_on = tmp;
812 812 allocsize += MAXPATHLEN;
813 813 }
814 814 bcopy(libname, mp->depends_on + osize, lsize);
815 815 *(mp->depends_on + nsize) = ' '; /* separate */
816 816 nsize++;
817 817 osize = nsize;
818 818 }
819 819 if (nsize) {
820 820 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
821 821 /*
822 822 * alloc with exact size and copy whatever it got over
823 823 */
824 824 KOBJ_MARK("realloc depends_on");
825 825 tmp = kobj_alloc(nsize, KM_WAIT);
826 826 bcopy(mp->depends_on, tmp, nsize);
827 827 kobj_free(mp->depends_on, allocsize);
828 828 mp->depends_on = tmp;
829 829 } else {
830 830 kobj_free(mp->depends_on, allocsize);
831 831 mp->depends_on = NULL;
832 832 }
833 833
834 834 mp->flags = KOBJ_EXEC|KOBJ_PRIM; /* NOT a relocatable .o */
835 835 mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
836 836 /*
837 837 * We allocate our own table since we don't
838 838 * hash undefined references.
839 839 */
840 840 KOBJ_MARK("chains");
841 841 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
842 842 KOBJ_MARK("buckets");
843 843 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
844 844
845 845 mp->text = _text;
846 846 mp->data = _data;
847 847
848 848 mp->text_size = _etext - _text;
849 849 mp->data_size = _edata - _data;
850 850
851 851 cp->mod_text = mp->text;
852 852 cp->mod_text_size = mp->text_size;
853 853
854 854 mp->filename = cp->mod_filename;
855 855
856 856 #ifdef KOBJ_DEBUG
857 857 if (kobj_debug & D_LOADING) {
858 858 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
859 859 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
860 860 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
861 861 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
862 862 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
863 863 }
864 864 #endif /* KOBJ_DEBUG */
865 865
866 866 /*
867 867 * Insert symbols into the hash table.
868 868 */
869 869 for (i = 0; i < mp->nsyms; i++) {
870 870 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
871 871
872 872 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
873 873 continue;
874 874 #if defined(__sparc)
875 875 /*
876 876 * Register symbols are ignored in the kernel
877 877 */
878 878 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER)
879 879 continue;
880 880 #endif /* __sparc */
881 881
882 882 sym_insert(mp, mp->strings + sp->st_name, i);
883 883 }
884 884
885 885 KOBJ_MARK("load_exec done");
886 886 return (mp);
887 887 }
888 888
889 889 /*
890 890 * Set up the linker module (if it's compiled in, LDNAME is NULL)
891 891 */
892 892 static void
893 893 load_linker(val_t *bootaux)
894 894 {
895 895 struct module *kmp = (struct module *)kobj_modules->mod_mp;
896 896 struct module *mp;
897 897 struct modctl *cp;
898 898 int i;
899 899 Shdr *shp;
900 900 Sym *sp;
901 901 int shsize;
902 902 char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
903 903
904 904 /*
905 905 * On some architectures, krtld is compiled into the kernel.
906 906 */
907 907 if (dlname == NULL)
908 908 return;
909 909
910 910 cp = add_primary(dlname, KOBJ_LM_PRIMARY);
911 911
912 912 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
913 913
914 914 cp->mod_mp = mp;
915 915 mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
916 916 shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
917 917 mp->shdrs = kobj_alloc(shsize, KM_WAIT);
918 918 bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
919 919
920 920 for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
921 921 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
922 922
923 923 if (shp->sh_flags & SHF_ALLOC) {
924 924 if (shp->sh_flags & SHF_WRITE) {
925 925 if (mp->data == NULL)
926 926 mp->data = (char *)shp->sh_addr;
927 927 } else if (mp->text == NULL) {
928 928 mp->text = (char *)shp->sh_addr;
929 929 }
930 930 }
931 931 if (shp->sh_type == SHT_SYMTAB) {
932 932 mp->symtbl_section = i;
933 933 mp->symhdr = shp;
934 934 mp->symspace = mp->symtbl = (char *)shp->sh_addr;
935 935 }
936 936 }
937 937 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
938 938 mp->flags = KOBJ_INTERP|KOBJ_PRIM;
939 939 mp->strhdr = (Shdr *)
940 940 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
941 941 mp->strings = (char *)mp->strhdr->sh_addr;
942 942 mp->hashsize = kobj_gethashsize(mp->nsyms);
943 943
944 944 mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
945 945 (mp->hashsize + mp->nsyms) * sizeof (symid_t);
946 946
947 947 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
948 948 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
949 949
950 950 mp->bss = bootaux[BA_BSS].ba_val;
951 951 mp->bss_align = 0; /* pre-aligned during allocation */
952 952 mp->bss_size = (uintptr_t)_edata - mp->bss;
953 953 mp->text_size = _etext - mp->text;
954 954 mp->data_size = _edata - mp->data;
955 955 mp->filename = cp->mod_filename;
956 956 cp->mod_text = mp->text;
957 957 cp->mod_text_size = mp->text_size;
958 958
959 959 /*
960 960 * Now that we've figured out where the linker is,
961 961 * set the limits for the booted object.
962 962 */
963 963 kmp->text_size = (size_t)(mp->text - kmp->text);
964 964 kmp->data_size = (size_t)(mp->data - kmp->data);
965 965 kobj_modules->mod_text_size = kmp->text_size;
966 966
967 967 #ifdef KOBJ_DEBUG
968 968 if (kobj_debug & D_LOADING) {
969 969 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
970 970 _kobj_printf(ops, "\ttext:0x%p", mp->text);
971 971 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
972 972 _kobj_printf(ops, "\tdata:0x%p", mp->data);
973 973 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
974 974 }
975 975 #endif /* KOBJ_DEBUG */
976 976
977 977 /*
978 978 * Insert the symbols into the hash table.
979 979 */
980 980 for (i = 0; i < mp->nsyms; i++) {
981 981 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
982 982
983 983 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
984 984 continue;
985 985 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
986 986 if (sp->st_shndx == SHN_COMMON)
987 987 sp->st_shndx = SHN_ABS;
988 988 }
989 989 sym_insert(mp, mp->strings + sp->st_name, i);
990 990 }
991 991
992 992 }
993 993
994 994 static kobj_notify_list_t **
995 995 kobj_notify_lookup(uint_t type)
996 996 {
997 997 ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
998 998 sizeof (kobj_notify_list_t *));
999 999
1000 1000 return (&kobj_notifiers[type]);
1001 1001 }
1002 1002
1003 1003 int
1004 1004 kobj_notify_add(kobj_notify_list_t *knp)
1005 1005 {
1006 1006 kobj_notify_list_t **knl;
1007 1007
1008 1008 knl = kobj_notify_lookup(knp->kn_type);
1009 1009
1010 1010 knp->kn_next = NULL;
1011 1011 knp->kn_prev = NULL;
1012 1012
1013 1013 mutex_enter(&kobj_lock);
1014 1014
1015 1015 if (*knl != NULL) {
1016 1016 (*knl)->kn_prev = knp;
1017 1017 knp->kn_next = *knl;
1018 1018 }
1019 1019 (*knl) = knp;
1020 1020
1021 1021 mutex_exit(&kobj_lock);
1022 1022 return (0);
1023 1023 }
1024 1024
1025 1025 int
1026 1026 kobj_notify_remove(kobj_notify_list_t *knp)
1027 1027 {
1028 1028 kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
1029 1029 kobj_notify_list_t *tknp;
1030 1030
1031 1031 mutex_enter(&kobj_lock);
1032 1032
1033 1033 /* LINTED */
1034 1034 if (tknp = knp->kn_next)
1035 1035 tknp->kn_prev = knp->kn_prev;
1036 1036
1037 1037 /* LINTED */
1038 1038 if (tknp = knp->kn_prev)
1039 1039 tknp->kn_next = knp->kn_next;
1040 1040 else
1041 1041 *knl = knp->kn_next;
1042 1042
1043 1043 mutex_exit(&kobj_lock);
1044 1044
1045 1045 return (0);
1046 1046 }
1047 1047
1048 1048 /*
1049 1049 * Notify all interested callbacks of a specified change in module state.
1050 1050 */
1051 1051 static void
1052 1052 kobj_notify(int type, struct modctl *modp)
1053 1053 {
1054 1054 kobj_notify_list_t *knp;
1055 1055
1056 1056 if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1057 1057 return;
1058 1058
1059 1059 mutex_enter(&kobj_lock);
1060 1060
1061 1061 for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1062 1062 knp->kn_func(type, modp);
1063 1063
1064 1064 /*
1065 1065 * KDI notification must be last (it has to allow for work done by the
1066 1066 * other notification callbacks), so we call it manually.
1067 1067 */
1068 1068 kobj_kdi_mod_notify(type, modp);
1069 1069
1070 1070 mutex_exit(&kobj_lock);
1071 1071 }
1072 1072
1073 1073 /*
1074 1074 * Create the module path.
1075 1075 */
1076 1076 static char *
1077 1077 getmodpath(const char *filename)
1078 1078 {
1079 1079 char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1080 1080
1081 1081 /*
1082 1082 * Platform code gets first crack, then add
1083 1083 * the default components
1084 1084 */
1085 1085 mach_modpath(path, filename);
1086 1086 if (*path != '\0')
1087 1087 (void) strcat(path, " ");
1088 1088 return (strcat(path, MOD_DEFPATH));
1089 1089 }
1090 1090
1091 1091 static struct modctl *
1092 1092 add_primary(const char *filename, int lmid)
1093 1093 {
1094 1094 struct modctl *cp;
1095 1095
1096 1096 cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1097 1097
1098 1098 cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1099 1099
1100 1100 /*
1101 1101 * For symbol lookup, we assemble our own
1102 1102 * modctl list of the primary modules.
1103 1103 */
1104 1104
1105 1105 (void) strcpy(cp->mod_filename, filename);
1106 1106 cp->mod_modname = basename(cp->mod_filename);
1107 1107
1108 1108 /* set values for modinfo assuming that the load will work */
1109 1109 cp->mod_prim = 1;
1110 1110 cp->mod_loaded = 1;
1111 1111 cp->mod_installed = 1;
1112 1112 cp->mod_loadcnt = 1;
1113 1113 cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1114 1114
1115 1115 cp->mod_id = kobj_last_module_id++;
1116 1116
1117 1117 /*
1118 1118 * Link the module in. We'll pass this info on
1119 1119 * to the mod squad later.
1120 1120 */
1121 1121 if (kobj_modules == NULL) {
1122 1122 kobj_modules = cp;
1123 1123 cp->mod_prev = cp->mod_next = cp;
1124 1124 } else {
1125 1125 cp->mod_prev = kobj_modules->mod_prev;
1126 1126 cp->mod_next = kobj_modules;
1127 1127 kobj_modules->mod_prev->mod_next = cp;
1128 1128 kobj_modules->mod_prev = cp;
1129 1129 }
1130 1130
1131 1131 kobj_lm_append(lmid, cp);
1132 1132
1133 1133 return (cp);
1134 1134 }
1135 1135
1136 1136 static int
1137 1137 bind_primary(val_t *bootaux, int lmid)
1138 1138 {
1139 1139 struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1140 1140 struct modctl_list *lp;
1141 1141 struct module *mp;
1142 1142
1143 1143 /*
1144 1144 * Do common symbols.
1145 1145 */
1146 1146 for (lp = linkmap; lp; lp = lp->modl_next) {
1147 1147 mp = mod(lp);
1148 1148
1149 1149 /*
1150 1150 * Don't do common section relocations for modules that
1151 1151 * don't need it.
1152 1152 */
1153 1153 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1154 1154 continue;
1155 1155
1156 1156 if (do_common(mp) < 0)
1157 1157 return (-1);
1158 1158 }
1159 1159
1160 1160 /*
1161 1161 * Resolve symbols.
1162 1162 */
1163 1163 for (lp = linkmap; lp; lp = lp->modl_next) {
1164 1164 mp = mod(lp);
1165 1165
1166 1166 if (do_symbols(mp, 0) < 0)
1167 1167 return (-1);
1168 1168 }
1169 1169
1170 1170 /*
1171 1171 * Do relocations.
1172 1172 */
1173 1173 for (lp = linkmap; lp; lp = lp->modl_next) {
1174 1174 mp = mod(lp);
1175 1175
1176 1176 if (mp->flags & KOBJ_EXEC) {
1177 1177 Dyn *dyn;
1178 1178 Word relasz = 0, relaent = 0;
1179 1179 Word shtype;
1180 1180 char *rela = NULL;
1181 1181
1182 1182 for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1183 1183 dyn->d_tag != DT_NULL; dyn++) {
1184 1184 switch (dyn->d_tag) {
1185 1185 case DT_RELASZ:
1186 1186 case DT_RELSZ:
1187 1187 relasz = dyn->d_un.d_val;
1188 1188 break;
1189 1189 case DT_RELAENT:
1190 1190 case DT_RELENT:
1191 1191 relaent = dyn->d_un.d_val;
1192 1192 break;
1193 1193 case DT_RELA:
1194 1194 shtype = SHT_RELA;
1195 1195 rela = (char *)dyn->d_un.d_ptr;
1196 1196 break;
1197 1197 case DT_REL:
1198 1198 shtype = SHT_REL;
1199 1199 rela = (char *)dyn->d_un.d_ptr;
1200 1200 break;
1201 1201 }
1202 1202 }
1203 1203 if (relasz == 0 ||
1204 1204 relaent == 0 || rela == NULL) {
1205 1205 _kobj_printf(ops, "krtld: bind_primary(): "
1206 1206 "no relocation information found for "
1207 1207 "module %s\n", mp->filename);
1208 1208 return (-1);
1209 1209 }
1210 1210 #ifdef KOBJ_DEBUG
1211 1211 if (kobj_debug & D_RELOCATIONS)
1212 1212 _kobj_printf(ops, "krtld: relocating: file=%s "
1213 1213 "KOBJ_EXEC\n", mp->filename);
1214 1214 #endif
1215 1215 if (do_relocate(mp, rela, shtype, relasz/relaent,
1216 1216 relaent, (Addr)mp->text) < 0)
1217 1217 return (-1);
1218 1218 } else {
1219 1219 if (do_relocations(mp) < 0)
1220 1220 return (-1);
1221 1221 }
1222 1222
1223 1223 kobj_sync_instruction_memory(mp->text, mp->text_size);
1224 1224 }
1225 1225
1226 1226 for (lp = linkmap; lp; lp = lp->modl_next) {
1227 1227 mp = mod(lp);
1228 1228
1229 1229 /*
1230 1230 * We need to re-read the full symbol table for the boot file,
1231 1231 * since we couldn't use the full one before. We also need to
1232 1232 * load the CTF sections of both the boot file and the
1233 1233 * interpreter (us).
1234 1234 */
1235 1235 if (mp->flags & KOBJ_EXEC) {
1236 1236 struct _buf *file;
1237 1237 int n;
1238 1238
1239 1239 file = kobj_open_file(mp->filename);
1240 1240 if (file == (struct _buf *)-1)
1241 1241 return (-1);
1242 1242 if (kobj_read_file(file, (char *)&mp->hdr,
1243 1243 sizeof (mp->hdr), 0) < 0)
1244 1244 return (-1);
1245 1245 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1246 1246 mp->shdrs = kobj_alloc(n, KM_WAIT);
1247 1247 if (kobj_read_file(file, mp->shdrs, n,
1248 1248 mp->hdr.e_shoff) < 0)
1249 1249 return (-1);
1250 1250 if (get_syms(mp, file) < 0)
1251 1251 return (-1);
1252 1252 if (get_ctf(mp, file) < 0)
1253 1253 return (-1);
1254 1254 kobj_close_file(file);
1255 1255 mp->flags |= KOBJ_RELOCATED;
1256 1256
1257 1257 } else if (mp->flags & KOBJ_INTERP) {
1258 1258 struct _buf *file;
1259 1259
1260 1260 /*
1261 1261 * The interpreter path fragment in mp->filename
1262 1262 * will already have the module directory suffix
1263 1263 * in it (if appropriate).
1264 1264 */
1265 1265 file = kobj_open_path(mp->filename, 1, 0);
1266 1266 if (file == (struct _buf *)-1)
1267 1267 return (-1);
1268 1268 if (get_ctf(mp, file) < 0)
1269 1269 return (-1);
1270 1270 kobj_close_file(file);
1271 1271 mp->flags |= KOBJ_RELOCATED;
1272 1272 }
1273 1273 }
1274 1274
1275 1275 return (0);
1276 1276 }
1277 1277
1278 1278 static struct modctl *
1279 1279 mod_already_loaded(char *modname)
1280 1280 {
1281 1281 struct modctl *mctl = kobj_modules;
1282 1282
1283 1283 do {
1284 1284 if (strcmp(modname, mctl->mod_filename) == 0)
1285 1285 return (mctl);
1286 1286 mctl = mctl->mod_next;
1287 1287
1288 1288 } while (mctl != kobj_modules);
1289 1289
1290 1290 return (NULL);
1291 1291 }
1292 1292
1293 1293 /*
1294 1294 * Load all the primary dependent modules.
1295 1295 */
1296 1296 static int
1297 1297 load_primary(struct module *mp, int lmid)
1298 1298 {
1299 1299 struct modctl *cp;
1300 1300 struct module *dmp;
1301 1301 char *p, *q;
1302 1302 char modname[MODMAXNAMELEN];
1303 1303
1304 1304 if ((p = mp->depends_on) == NULL)
1305 1305 return (0);
1306 1306
1307 1307 /* CONSTANTCONDITION */
1308 1308 while (1) {
1309 1309 /*
1310 1310 * Skip space.
1311 1311 */
1312 1312 while (*p && (*p == ' ' || *p == '\t'))
1313 1313 p++;
1314 1314 /*
1315 1315 * Get module name.
1316 1316 */
1317 1317 q = modname;
1318 1318 while (*p && *p != ' ' && *p != '\t')
1319 1319 *q++ = *p++;
1320 1320
1321 1321 if (q == modname)
1322 1322 break;
1323 1323
1324 1324 *q = '\0';
1325 1325 /*
1326 1326 * Check for dup dependencies.
1327 1327 */
1328 1328 if (strcmp(modname, "dtracestubs") == 0 ||
1329 1329 mod_already_loaded(modname) != NULL)
1330 1330 continue;
1331 1331
1332 1332 cp = add_primary(modname, lmid);
1333 1333 cp->mod_busy = 1;
1334 1334 /*
1335 1335 * Load it.
1336 1336 */
1337 1337 (void) kobj_load_module(cp, 1);
1338 1338 cp->mod_busy = 0;
1339 1339
1340 1340 if ((dmp = cp->mod_mp) == NULL) {
1341 1341 cp->mod_loaded = 0;
1342 1342 cp->mod_installed = 0;
1343 1343 cp->mod_loadcnt = 0;
1344 1344 return (-1);
1345 1345 }
1346 1346
1347 1347 add_dependent(mp, dmp);
1348 1348 dmp->flags |= KOBJ_PRIM;
1349 1349
1350 1350 /*
1351 1351 * Recurse.
1352 1352 */
1353 1353 if (load_primary(dmp, lmid) == -1) {
1354 1354 cp->mod_loaded = 0;
1355 1355 cp->mod_installed = 0;
1356 1356 cp->mod_loadcnt = 0;
1357 1357 return (-1);
1358 1358 }
1359 1359 }
1360 1360 return (0);
1361 1361 }
1362 1362
1363 1363 static int
1364 1364 console_is_usb_serial(void)
1365 1365 {
1366 1366 char *console;
1367 1367 int len, ret;
1368 1368
1369 1369 if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1370 1370 return (0);
1371 1371
1372 1372 console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1373 1373 (void) BOP_GETPROP(ops, "console", console);
1374 1374 ret = (strcmp(console, "usb-serial") == 0);
1375 1375 kobj_free(console, len);
1376 1376
1377 1377 return (ret);
1378 1378 }
1379 1379
1380 1380 static int
1381 1381 load_kmdb(val_t *bootaux)
1382 1382 {
1383 1383 struct modctl *mctl;
1384 1384 struct module *mp;
1385 1385 Sym *sym;
1386 1386
1387 1387 if (console_is_usb_serial()) {
1388 1388 _kobj_printf(ops, "kmdb not loaded "
1389 1389 "(unsupported on usb serial console)\n");
1390 1390 return (0);
1391 1391 }
1392 1392
1393 1393 _kobj_printf(ops, "Loading kmdb...\n");
1394 1394
1395 1395 if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1396 1396 return (-1);
1397 1397
1398 1398 mctl->mod_busy = 1;
1399 1399 (void) kobj_load_module(mctl, 1);
1400 1400 mctl->mod_busy = 0;
1401 1401
1402 1402 if ((mp = mctl->mod_mp) == NULL)
1403 1403 return (-1);
1404 1404
1405 1405 mp->flags |= KOBJ_PRIM;
1406 1406
1407 1407 if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1408 1408 return (-1);
1409 1409
1410 1410 if (boothowto & RB_VERBOSE)
1411 1411 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1412 1412
1413 1413 if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1414 1414 return (-1);
1415 1415
1416 1416 if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1417 1417 return (-1);
1418 1418
1419 1419 #ifdef KOBJ_DEBUG
1420 1420 if (kobj_debug & D_DEBUG) {
1421 1421 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1422 1422 sym->st_value);
1423 1423 _kobj_printf(ops, "\tops 0x%p\n", ops);
1424 1424 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1425 1425 }
1426 1426 #endif
1427 1427
1428 1428 if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1429 1429 (const char **)kobj_kmdb_argv) < 0)
1430 1430 return (-1);
1431 1431
1432 1432 return (0);
1433 1433 }
1434 1434
1435 1435 /*
1436 1436 * Return a string listing module dependencies.
1437 1437 */
1438 1438 static char *
1439 1439 depends_on(struct module *mp)
1440 1440 {
1441 1441 Sym *sp;
1442 1442 char *depstr, *q;
1443 1443
1444 1444 /*
1445 1445 * The module doesn't have a depends_on value, so let's try it the
1446 1446 * old-fashioned way - via "_depends_on"
1447 1447 */
1448 1448 if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1449 1449 return (NULL);
1450 1450
1451 1451 q = (char *)sp->st_value;
1452 1452
1453 1453 #ifdef KOBJ_DEBUG
1454 1454 /*
1455 1455 * _depends_on is a deprecated interface, so we warn about its use
1456 1456 * irrespective of subsequent processing errors. How else are we going
1457 1457 * to be able to deco this interface completely?
1458 1458 * Changes initially limited to DEBUG because third-party modules
1459 1459 * should be flagged to developers before general use base.
1460 1460 */
1461 1461 _kobj_printf(ops,
1462 1462 "Warning: %s uses deprecated _depends_on interface.\n",
1463 1463 mp->filename);
1464 1464 _kobj_printf(ops, "Please notify module developer or vendor.\n");
1465 1465 #endif
1466 1466
1467 1467 /*
1468 1468 * Idiot checks. Make sure it's
1469 1469 * in-bounds and NULL terminated.
1470 1470 */
1471 1471 if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1472 1472 _kobj_printf(ops, "Error processing dependency for %s\n",
1473 1473 mp->filename);
1474 1474 return (NULL);
1475 1475 }
1476 1476
1477 1477 depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1478 1478 (void) strcpy(depstr, q);
1479 1479
1480 1480 return (depstr);
1481 1481 }
1482 1482
1483 1483 void
1484 1484 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1485 1485 {
1486 1486 struct module *mp;
1487 1487 mp = (struct module *)xmp;
1488 1488
1489 1489 modinfo->mi_base = mp->text;
1490 1490 modinfo->mi_size = mp->text_size + mp->data_size;
1491 1491 }
1492 1492
1493 1493 /*
1494 1494 * kobj_export_ksyms() performs the following services:
1495 1495 *
1496 1496 * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1497 1497 * (2) Removes unneeded symbols to save space.
1498 1498 * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1499 1499 * (4) Makes the symbol table visible to /dev/ksyms.
1500 1500 */
1501 1501 static void
1502 1502 kobj_export_ksyms(struct module *mp)
1503 1503 {
1504 1504 Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1505 1505 Sym *sp, *osp;
1506 1506 char *name;
1507 1507 size_t namelen;
1508 1508 struct module *omp;
1509 1509 uint_t nsyms;
1510 1510 size_t symsize = mp->symhdr->sh_entsize;
1511 1511 size_t locals = 1;
1512 1512 size_t strsize;
1513 1513
1514 1514 /*
1515 1515 * Make a copy of the original module structure.
1516 1516 */
1517 1517 omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1518 1518 bcopy(mp, omp, sizeof (struct module));
1519 1519
1520 1520 /*
1521 1521 * Compute the sizes of the new symbol table sections.
1522 1522 */
1523 1523 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1524 1524 if (osp->st_value == 0)
1525 1525 continue;
1526 1526 if (sym_lookup(omp, osp) == NULL)
1527 1527 continue;
1528 1528 name = omp->strings + osp->st_name;
1529 1529 namelen = strlen(name);
1530 1530 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1531 1531 locals++;
1532 1532 nsyms++;
1533 1533 strsize += namelen + 1;
1534 1534 }
1535 1535
1536 1536 mp->nsyms = nsyms;
1537 1537 mp->hashsize = kobj_gethashsize(mp->nsyms);
1538 1538
1539 1539 /*
1540 1540 * ksyms_lock must be held as writer during any operation that
1541 1541 * modifies ksyms_arena, including allocation from same, and
1542 1542 * must not be dropped until the arena is vmem_walk()able.
1543 1543 */
1544 1544 rw_enter(&ksyms_lock, RW_WRITER);
1545 1545
1546 1546 /*
1547 1547 * Allocate space for the new section headers (symtab and strtab),
1548 1548 * symbol table, buckets, chains, and strings.
1549 1549 */
1550 1550 mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1551 1551 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1552 1552
1553 1553 if (mp->flags & KOBJ_NOKSYMS) {
1554 1554 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1555 1555 } else {
1556 1556 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1557 1557 VM_BESTFIT | VM_SLEEP);
1558 1558 }
1559 1559 bzero(mp->symspace, mp->symsize);
1560 1560
1561 1561 /*
1562 1562 * Divvy up symspace.
1563 1563 */
1564 1564 mp->shdrs = mp->symspace;
1565 1565 mp->symhdr = (Shdr *)mp->shdrs;
1566 1566 mp->strhdr = (Shdr *)(mp->symhdr + 1);
1567 1567 mp->symtbl = (char *)(mp->strhdr + 1);
1568 1568 mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1569 1569 mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1570 1570 mp->strings = (char *)(mp->chains + nsyms);
1571 1571
1572 1572 /*
1573 1573 * Fill in the new section headers (symtab and strtab).
1574 1574 */
1575 1575 mp->hdr.e_shnum = 2;
1576 1576 mp->symtbl_section = 0;
1577 1577
1578 1578 mp->symhdr->sh_type = SHT_SYMTAB;
1579 1579 mp->symhdr->sh_addr = (Addr)mp->symtbl;
1580 1580 mp->symhdr->sh_size = nsyms * symsize;
1581 1581 mp->symhdr->sh_link = 1;
1582 1582 mp->symhdr->sh_info = locals;
1583 1583 mp->symhdr->sh_addralign = sizeof (Addr);
1584 1584 mp->symhdr->sh_entsize = symsize;
1585 1585
1586 1586 mp->strhdr->sh_type = SHT_STRTAB;
1587 1587 mp->strhdr->sh_addr = (Addr)mp->strings;
1588 1588 mp->strhdr->sh_size = strsize;
1589 1589 mp->strhdr->sh_addralign = 1;
1590 1590
1591 1591 /*
1592 1592 * Construct the new symbol table.
1593 1593 */
1594 1594 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1595 1595 if (osp->st_value == 0)
1596 1596 continue;
1597 1597 if (sym_lookup(omp, osp) == NULL)
1598 1598 continue;
1599 1599 name = omp->strings + osp->st_name;
1600 1600 namelen = strlen(name);
1601 1601 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1602 1602 bcopy(osp, sp, symsize);
1603 1603 bcopy(name, mp->strings + strsize, namelen);
1604 1604 sp->st_name = strsize;
1605 1605 sym_insert(mp, name, nsyms);
1606 1606 nsyms++;
1607 1607 strsize += namelen + 1;
1608 1608 }
1609 1609
1610 1610 rw_exit(&ksyms_lock);
1611 1611
1612 1612 /*
1613 1613 * Free the old section headers -- we'll never need them again.
1614 1614 */
1615 1615 if (!(mp->flags & KOBJ_PRIM)) {
1616 1616 uint_t shn;
1617 1617 Shdr *shp;
1618 1618
1619 1619 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1620 1620 shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1621 1621 switch (shp->sh_type) {
1622 1622 case SHT_RELA:
1623 1623 case SHT_REL:
1624 1624 if (shp->sh_addr != 0) {
1625 1625 kobj_free((void *)shp->sh_addr,
1626 1626 shp->sh_size);
1627 1627 }
1628 1628 break;
1629 1629 }
1630 1630 }
1631 1631 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1632 1632 }
1633 1633 /*
1634 1634 * Discard the old symbol table and our copy of the module strucure.
1635 1635 */
1636 1636 if (!(mp->flags & KOBJ_PRIM))
1637 1637 kobj_free(omp->symspace, omp->symsize);
1638 1638 kobj_free(omp, sizeof (struct module));
1639 1639 }
1640 1640
1641 1641 static void
1642 1642 kobj_export_ctf(struct module *mp)
1643 1643 {
1644 1644 char *data = mp->ctfdata;
1645 1645 size_t size = mp->ctfsize;
1646 1646
1647 1647 if (data != NULL) {
1648 1648 if (_moddebug & MODDEBUG_NOCTF) {
1649 1649 mp->ctfdata = NULL;
1650 1650 mp->ctfsize = 0;
1651 1651 } else {
1652 1652 mp->ctfdata = vmem_alloc(ctf_arena, size,
1653 1653 VM_BESTFIT | VM_SLEEP);
1654 1654 bcopy(data, mp->ctfdata, size);
1655 1655 }
1656 1656
1657 1657 if (!(mp->flags & KOBJ_PRIM))
1658 1658 kobj_free(data, size);
1659 1659 }
1660 1660 }
1661 1661
1662 1662 void
1663 1663 kobj_export_module(struct module *mp)
1664 1664 {
1665 1665 kobj_export_ksyms(mp);
1666 1666 kobj_export_ctf(mp);
1667 1667
1668 1668 mp->flags |= KOBJ_EXPORTED;
1669 1669 }
1670 1670
1671 1671 static int
1672 1672 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1673 1673 {
1674 1674 char *path = NULL, *depstr = NULL;
1675 1675 int allocsize = 0, osize = 0, nsize = 0;
1676 1676 char *libname, *tmp;
1677 1677 int lsize;
1678 1678 Dyn *dynp;
1679 1679
1680 1680 for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1681 1681 switch (dynp->d_tag) {
1682 1682 case DT_NEEDED:
1683 1683 /*
1684 1684 * Read the DT_NEEDED entries, expanding the macros they
1685 1685 * contain (if any), and concatenating them into a
1686 1686 * single space-separated dependency list.
1687 1687 */
1688 1688 libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1689 1689
1690 1690 if (strchr(libname, '$') != NULL) {
1691 1691 char *_lib;
1692 1692
1693 1693 if (path == NULL)
1694 1694 path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1695 1695 if ((_lib = expand_libmacro(libname, path,
1696 1696 path)) != NULL)
1697 1697 libname = _lib;
1698 1698 else {
1699 1699 _kobj_printf(ops, "krtld: "
1700 1700 "process_dynamic: failed to expand "
1701 1701 "%s\n", libname);
1702 1702 }
1703 1703 }
1704 1704
1705 1705 lsize = strlen(libname);
1706 1706 nsize += lsize;
1707 1707 if (nsize + 1 > allocsize) {
1708 1708 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1709 1709 KM_WAIT);
1710 1710 if (depstr != NULL) {
1711 1711 bcopy(depstr, tmp, osize);
1712 1712 kobj_free(depstr, allocsize);
1713 1713 }
1714 1714 depstr = tmp;
1715 1715 allocsize += MAXPATHLEN;
1716 1716 }
1717 1717 bcopy(libname, depstr + osize, lsize);
1718 1718 *(depstr + nsize) = ' '; /* separator */
1719 1719 nsize++;
1720 1720 osize = nsize;
1721 1721 break;
1722 1722
1723 1723 case DT_FLAGS_1:
1724 1724 if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1725 1725 mp->flags |= KOBJ_IGNMULDEF;
1726 1726 if (dynp->d_un.d_val & DF_1_NOKSYMS)
1727 1727 mp->flags |= KOBJ_NOKSYMS;
1728 1728
1729 1729 break;
1730 1730 }
1731 1731 }
1732 1732
1733 1733 /*
1734 1734 * finish up the depends string (if any)
1735 1735 */
1736 1736 if (depstr != NULL) {
1737 1737 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1738 1738 if (path != NULL)
1739 1739 kobj_free(path, MAXPATHLEN);
1740 1740
1741 1741 tmp = kobj_alloc(nsize, KM_WAIT);
1742 1742 bcopy(depstr, tmp, nsize);
1743 1743 kobj_free(depstr, allocsize);
1744 1744 depstr = tmp;
1745 1745
1746 1746 mp->depends_on = depstr;
1747 1747 }
1748 1748
1749 1749 return (0);
1750 1750 }
1751 1751
1752 1752 static int
1753 1753 do_dynamic(struct module *mp, struct _buf *file)
1754 1754 {
1755 1755 Shdr *dshp, *dstrp, *shp;
1756 1756 char *dyndata, *dstrdata;
1757 1757 int dshn, shn, rc;
1758 1758
1759 1759 /* find and validate the dynamic section (if any) */
1760 1760
1761 1761 for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1762 1762 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1763 1763 switch (shp->sh_type) {
1764 1764 case SHT_DYNAMIC:
1765 1765 if (dshp != NULL) {
1766 1766 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1767 1767 mp->filename);
1768 1768 _kobj_printf(ops,
1769 1769 "multiple dynamic sections\n");
1770 1770 return (-1);
1771 1771 } else {
1772 1772 dshp = shp;
1773 1773 dshn = shn;
1774 1774 }
1775 1775 break;
1776 1776 }
1777 1777 }
1778 1778
1779 1779 if (dshp == NULL)
1780 1780 return (0);
1781 1781
1782 1782 if (dshp->sh_link > mp->hdr.e_shnum) {
1783 1783 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1784 1784 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1785 1785 return (-1);
1786 1786 }
1787 1787 dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1788 1788
1789 1789 if (dstrp->sh_type != SHT_STRTAB) {
1790 1790 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1791 1791 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1792 1792 dshn);
1793 1793 return (-1);
1794 1794 }
1795 1795
1796 1796 /* read it from disk */
1797 1797
1798 1798 dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1799 1799 if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1800 1800 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1801 1801 _kobj_printf(ops, "error reading section %d\n", dshn);
1802 1802
1803 1803 kobj_free(dyndata, dshp->sh_size);
1804 1804 return (-1);
1805 1805 }
1806 1806
1807 1807 dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1808 1808 if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1809 1809 dstrp->sh_offset) < 0) {
1810 1810 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1811 1811 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1812 1812
1813 1813 kobj_free(dyndata, dshp->sh_size);
1814 1814 kobj_free(dstrdata, dstrp->sh_size);
1815 1815 return (-1);
1816 1816 }
1817 1817
1818 1818 /* pull the interesting pieces out */
1819 1819
1820 1820 rc = process_dynamic(mp, dyndata, dstrdata);
1821 1821
1822 1822 kobj_free(dyndata, dshp->sh_size);
1823 1823 kobj_free(dstrdata, dstrp->sh_size);
1824 1824
1825 1825 return (rc);
1826 1826 }
1827 1827
1828 1828 void
1829 1829 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1830 1830 {
1831 1831 if (!standalone) {
1832 1832 if (mp->ctfdata != NULL) {
1833 1833 if (vmem_contains(ctf_arena, mp->ctfdata,
1834 1834 mp->ctfsize)) {
1835 1835 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1836 1836 } else {
1837 1837 kobj_free(mp->ctfdata, mp->ctfsize);
1838 1838 }
1839 1839 }
1840 1840 }
1841 1841
1842 1842 /*
1843 1843 * The order is very important here. We need to make sure that
1844 1844 * consumers, at any given instant, see a consistent state. We'd
1845 1845 * rather they see no CTF data than the address of one buffer and the
1846 1846 * size of another.
1847 1847 */
1848 1848 mp->ctfdata = NULL;
1849 1849 membar_producer();
1850 1850 mp->ctfsize = size;
1851 1851 mp->ctfdata = data;
1852 1852 membar_producer();
1853 1853 }
1854 1854
1855 1855 int
1856 1856 kobj_load_module(struct modctl *modp, int use_path)
1857 1857 {
1858 1858 char *filename = modp->mod_filename;
1859 1859 char *modname = modp->mod_modname;
1860 1860 int i;
1861 1861 int n;
1862 1862 struct _buf *file;
1863 1863 struct module *mp = NULL;
1864 1864 #ifdef MODDIR_SUFFIX
1865 1865 int no_suffixdir_drv = 0;
1866 1866 #endif
1867 1867
1868 1868 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1869 1869
1870 1870 /*
1871 1871 * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1872 1872 * kmdb contains a bunch of symbols with well-known names, symbols
1873 1873 * which will mask the real versions, thus causing no end of trouble
1874 1874 * for mdb.
1875 1875 */
1876 1876 if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1877 1877 mp->flags |= KOBJ_NOKSYMS;
1878 1878
1879 1879 file = kobj_open_path(filename, use_path, 1);
1880 1880 if (file == (struct _buf *)-1) {
1881 1881 #ifdef MODDIR_SUFFIX
1882 1882 file = kobj_open_path(filename, use_path, 0);
1883 1883 #endif
1884 1884 if (file == (struct _buf *)-1) {
1885 1885 kobj_free(mp, sizeof (*mp));
1886 1886 goto bad;
1887 1887 }
1888 1888 #ifdef MODDIR_SUFFIX
1889 1889 /*
1890 1890 * There is no driver module in the ISA specific (suffix)
1891 1891 * subdirectory but there is a module in the parent directory.
1892 1892 */
1893 1893 if (strncmp(filename, "drv/", 4) == 0) {
1894 1894 no_suffixdir_drv = 1;
1895 1895 }
1896 1896 #endif
1897 1897 }
1898 1898
1899 1899 mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1900 1900 (void) strcpy(mp->filename, file->_name);
1901 1901
1902 1902 if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1903 1903 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1904 1904 modname);
1905 1905 kobj_free(mp->filename, strlen(file->_name) + 1);
1906 1906 kobj_free(mp, sizeof (*mp));
1907 1907 goto bad;
1908 1908 }
1909 1909 for (i = 0; i < SELFMAG; i++) {
1910 1910 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1911 1911 if (_moddebug & MODDEBUG_ERRMSG)
1912 1912 _kobj_printf(ops, "%s not an elf module\n",
1913 1913 modname);
1914 1914 kobj_free(mp->filename, strlen(file->_name) + 1);
1915 1915 kobj_free(mp, sizeof (*mp));
1916 1916 goto bad;
1917 1917 }
1918 1918 }
1919 1919 /*
1920 1920 * It's ELF, but is it our ISA? Interpreting the header
1921 1921 * from a file for a byte-swapped ISA could cause a huge
1922 1922 * and unsatisfiable value to be passed to kobj_alloc below
1923 1923 * and therefore hang booting.
1924 1924 */
1925 1925 if (!elf_mach_ok(&mp->hdr)) {
1926 1926 if (_moddebug & MODDEBUG_ERRMSG)
1927 1927 _kobj_printf(ops, "%s not an elf module for this ISA\n",
1928 1928 modname);
1929 1929 kobj_free(mp->filename, strlen(file->_name) + 1);
1930 1930 kobj_free(mp, sizeof (*mp));
1931 1931 #ifdef MODDIR_SUFFIX
1932 1932 /*
1933 1933 * The driver mod is not in the ISA specific subdirectory
1934 1934 * and the module in the parent directory is not our ISA.
1935 1935 * If it is our ISA, for now we will silently succeed.
1936 1936 */
1937 1937 if (no_suffixdir_drv == 1) {
1938 1938 cmn_err(CE_CONT, "?NOTICE: %s: 64-bit driver module"
1939 1939 " not found\n", modname);
1940 1940 }
1941 1941 #endif
1942 1942 goto bad;
1943 1943 }
1944 1944
1945 1945 /*
1946 1946 * All modules, save for unix, should be relocatable (as opposed to
1947 1947 * dynamic). Dynamic modules come with PLTs and GOTs, which can't
1948 1948 * currently be processed by krtld.
1949 1949 */
1950 1950 if (mp->hdr.e_type != ET_REL) {
1951 1951 if (_moddebug & MODDEBUG_ERRMSG)
1952 1952 _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1953 1953 "module\n", modname);
1954 1954 kobj_free(mp->filename, strlen(file->_name) + 1);
1955 1955 kobj_free(mp, sizeof (*mp));
1956 1956 goto bad;
1957 1957 }
1958 1958
1959 1959 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1960 1960 mp->shdrs = kobj_alloc(n, KM_WAIT);
1961 1961
1962 1962 if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1963 1963 _kobj_printf(ops, "kobj_load_module: %s error reading "
1964 1964 "section headers\n", modname);
1965 1965 kobj_free(mp->shdrs, n);
1966 1966 kobj_free(mp->filename, strlen(file->_name) + 1);
1967 1967 kobj_free(mp, sizeof (*mp));
1968 1968 goto bad;
1969 1969 }
1970 1970
1971 1971 kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1972 1972 module_assign(modp, mp);
1973 1973
1974 1974 /* read in sections */
1975 1975 if (get_progbits(mp, file) < 0) {
1976 1976 _kobj_printf(ops, "%s error reading sections\n", modname);
1977 1977 goto bad;
1978 1978 }
1979 1979
1980 1980 if (do_dynamic(mp, file) < 0) {
1981 1981 _kobj_printf(ops, "%s error reading dynamic section\n",
1982 1982 modname);
1983 1983 goto bad;
1984 1984 }
1985 1985
1986 1986 modp->mod_text = mp->text;
1987 1987 modp->mod_text_size = mp->text_size;
1988 1988
1989 1989 /* read in symbols; adjust values for each section's real address */
1990 1990 if (get_syms(mp, file) < 0) {
1991 1991 _kobj_printf(ops, "%s error reading symbols\n",
1992 1992 modname);
1993 1993 goto bad;
1994 1994 }
1995 1995
1996 1996 /*
1997 1997 * If we didn't dependency information from the dynamic section, look
1998 1998 * for it the old-fashioned way.
1999 1999 */
2000 2000 if (mp->depends_on == NULL)
2001 2001 mp->depends_on = depends_on(mp);
2002 2002
2003 2003 if (get_ctf(mp, file) < 0) {
2004 2004 _kobj_printf(ops, "%s debug information will not "
2005 2005 "be available\n", modname);
2006 2006 }
2007 2007
2008 2008 /* primary kernel modules do not have a signature section */
2009 2009 if (!(mp->flags & KOBJ_PRIM))
2010 2010 get_signature(mp, file);
2011 2011
2012 2012 #ifdef KOBJ_DEBUG
2013 2013 if (kobj_debug & D_LOADING) {
2014 2014 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
2015 2015 _kobj_printf(ops, "\ttext:0x%p", mp->text);
2016 2016 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
2017 2017 _kobj_printf(ops, "\tdata:0x%p", mp->data);
2018 2018 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
2019 2019 }
2020 2020 #endif /* KOBJ_DEBUG */
2021 2021
2022 2022 /*
2023 2023 * For primary kernel modules, we defer
2024 2024 * symbol resolution and relocation until
2025 2025 * all primary objects have been loaded.
2026 2026 */
2027 2027 if (!standalone) {
2028 2028 int ddrval, dcrval;
2029 2029 char *dependent_modname;
2030 2030 /* load all dependents */
2031 2031 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
2032 2032 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
2033 2033
2034 2034 /*
2035 2035 * resolve undefined and common symbols,
2036 2036 * also allocates common space
2037 2037 */
2038 2038 if ((dcrval = do_common(mp)) < 0) {
2039 2039 switch (dcrval) {
2040 2040 case DOSYM_UNSAFE:
2041 2041 _kobj_printf(ops, "WARNING: mod_load: "
2042 2042 "MT-unsafe module '%s' rejected\n",
2043 2043 modname);
2044 2044 break;
2045 2045 case DOSYM_UNDEF:
2046 2046 _kobj_printf(ops, "WARNING: mod_load: "
2047 2047 "cannot load module '%s'\n",
2048 2048 modname);
2049 2049 if (ddrval == -1) {
2050 2050 _kobj_printf(ops, "WARNING: %s: ",
2051 2051 modname);
2052 2052 _kobj_printf(ops,
2053 2053 "unable to resolve dependency, "
2054 2054 "module '%s' not found\n",
2055 2055 dependent_modname);
2056 2056 }
2057 2057 break;
2058 2058 }
2059 2059 }
2060 2060 kobj_free(dependent_modname, MODMAXNAMELEN);
2061 2061 if (dcrval < 0)
2062 2062 goto bad;
2063 2063
2064 2064 /* process relocation tables */
2065 2065 if (do_relocations(mp) < 0) {
2066 2066 _kobj_printf(ops, "%s error doing relocations\n",
2067 2067 modname);
2068 2068 goto bad;
2069 2069 }
2070 2070
2071 2071 if (mp->destination) {
2072 2072 off_t off = (uintptr_t)mp->destination & PAGEOFFSET;
2073 2073 caddr_t base = (caddr_t)mp->destination - off;
2074 2074 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2075 2075
2076 2076 hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2077 2077 vmem_free(heap_arena, base, size);
2078 2078 }
2079 2079
2080 2080 /* sync_instruction_memory */
2081 2081 kobj_sync_instruction_memory(mp->text, mp->text_size);
2082 2082 kobj_export_module(mp);
2083 2083 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2084 2084 }
2085 2085 kobj_close_file(file);
2086 2086 return (0);
2087 2087 bad:
2088 2088 if (file != (struct _buf *)-1)
2089 2089 kobj_close_file(file);
2090 2090 if (modp->mod_mp != NULL)
2091 2091 free_module_data(modp->mod_mp);
2092 2092
2093 2093 module_assign(modp, NULL);
2094 2094 return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2095 2095 }
2096 2096
2097 2097 int
2098 2098 kobj_load_primary_module(struct modctl *modp)
2099 2099 {
2100 2100 struct modctl *dep;
2101 2101 struct module *mp;
2102 2102
2103 2103 if (kobj_load_module(modp, 0) != 0)
2104 2104 return (-1);
2105 2105
2106 2106 mp = modp->mod_mp;
2107 2107 mp->flags |= KOBJ_PRIM;
2108 2108
2109 2109 /* Bind new module to its dependents */
2110 2110 if (mp->depends_on != NULL && (dep =
2111 2111 mod_already_loaded(mp->depends_on)) == NULL) {
2112 2112 #ifdef KOBJ_DEBUG
2113 2113 if (kobj_debug & D_DEBUG) {
2114 2114 _kobj_printf(ops, "krtld: failed to resolve deps "
2115 2115 "for primary %s\n", modp->mod_modname);
2116 2116 }
2117 2117 #endif
2118 2118 return (-1);
2119 2119 }
2120 2120
2121 2121 add_dependent(mp, dep->mod_mp);
2122 2122
2123 2123 /*
2124 2124 * Relocate it. This module may not be part of a link map, so we
2125 2125 * can't use bind_primary.
2126 2126 */
2127 2127 if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2128 2128 do_relocations(mp) < 0) {
2129 2129 #ifdef KOBJ_DEBUG
2130 2130 if (kobj_debug & D_DEBUG) {
2131 2131 _kobj_printf(ops, "krtld: failed to relocate "
2132 2132 "primary %s\n", modp->mod_modname);
2133 2133 }
2134 2134 #endif
2135 2135 return (-1);
2136 2136 }
2137 2137
2138 2138 return (0);
2139 2139 }
2140 2140
2141 2141 static void
2142 2142 module_assign(struct modctl *cp, struct module *mp)
2143 2143 {
2144 2144 if (standalone) {
2145 2145 cp->mod_mp = mp;
2146 2146 return;
2147 2147 }
2148 2148 mutex_enter(&mod_lock);
2149 2149 cp->mod_mp = mp;
2150 2150 cp->mod_gencount++;
2151 2151 mutex_exit(&mod_lock);
2152 2152 }
2153 2153
2154 2154 void
2155 2155 kobj_unload_module(struct modctl *modp)
2156 2156 {
2157 2157 struct module *mp = modp->mod_mp;
2158 2158
2159 2159 if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2160 2160 _kobj_printf(ops, "text for %s ", mp->filename);
2161 2161 _kobj_printf(ops, "was at %p\n", mp->text);
2162 2162 mp->text = NULL; /* don't actually free it */
2163 2163 }
2164 2164
2165 2165 kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2166 2166
2167 2167 /*
2168 2168 * Null out mod_mp first, so consumers (debuggers) know not to look
2169 2169 * at the module structure any more.
2170 2170 */
2171 2171 mutex_enter(&mod_lock);
2172 2172 modp->mod_mp = NULL;
|
↓ open down ↓ |
2172 lines elided |
↑ open up ↑ |
2173 2173 mutex_exit(&mod_lock);
2174 2174
2175 2175 kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2176 2176 free_module_data(mp);
2177 2177 }
2178 2178
2179 2179 static void
2180 2180 free_module_data(struct module *mp)
2181 2181 {
2182 2182 struct module_list *lp, *tmp;
2183 + hotinline_desc_t *hid, *next;
2183 2184 int ksyms_exported = 0;
2184 2185
2185 2186 lp = mp->head;
2186 2187 while (lp) {
2187 2188 tmp = lp;
2188 2189 lp = lp->next;
2189 2190 kobj_free((char *)tmp, sizeof (*tmp));
2190 2191 }
2191 2192
2193 + /* release hotinlines */
2194 + hid = mp->hi_calls;
2195 + while (hid != NULL) {
2196 + next = hid->hid_next;
2197 + kobj_free(hid->hid_symname, strlen(hid->hid_symname) + 1);
2198 + kobj_free(hid, sizeof (hotinline_desc_t));
2199 + hid = next;
2200 + }
2201 +
2192 2202 rw_enter(&ksyms_lock, RW_WRITER);
2193 2203 if (mp->symspace) {
2194 2204 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2195 2205 vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2196 2206 ksyms_exported = 1;
2197 2207 } else {
2198 2208 if (mp->flags & KOBJ_NOKSYMS)
2199 2209 ksyms_exported = 1;
2200 2210 kobj_free(mp->symspace, mp->symsize);
2201 2211 }
2202 2212 }
2203 2213 rw_exit(&ksyms_lock);
2204 2214
2205 2215 if (mp->ctfdata) {
2206 2216 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2207 2217 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2208 2218 else
2209 2219 kobj_free(mp->ctfdata, mp->ctfsize);
2210 2220 }
2211 2221
2212 2222 if (mp->sigdata)
2213 2223 kobj_free(mp->sigdata, mp->sigsize);
2214 2224
2215 2225 /*
2216 2226 * We did not get far enough into kobj_export_ksyms() to free allocated
2217 2227 * buffers because we encounted error conditions. Free the buffers.
2218 2228 */
2219 2229 if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2220 2230 uint_t shn;
2221 2231 Shdr *shp;
2222 2232
2223 2233 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2224 2234 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2225 2235 switch (shp->sh_type) {
2226 2236 case SHT_RELA:
2227 2237 case SHT_REL:
2228 2238 if (shp->sh_addr != 0)
2229 2239 kobj_free((void *)shp->sh_addr,
2230 2240 shp->sh_size);
2231 2241 break;
2232 2242 }
2233 2243 }
2234 2244 err_free_done:
2235 2245 if (!(mp->flags & KOBJ_PRIM)) {
2236 2246 kobj_free(mp->shdrs,
2237 2247 mp->hdr.e_shentsize * mp->hdr.e_shnum);
2238 2248 }
2239 2249 }
2240 2250
2241 2251 if (mp->bss)
2242 2252 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2243 2253
2244 2254 if (mp->fbt_tab)
2245 2255 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2246 2256
2247 2257 if (mp->textwin_base)
2248 2258 kobj_textwin_free(mp);
2249 2259
2250 2260 if (mp->sdt_probes != NULL) {
2251 2261 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2252 2262
2253 2263 while (sdp != NULL) {
2254 2264 next = sdp->sdpd_next;
2255 2265 kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2256 2266 kobj_free(sdp, sizeof (sdt_probedesc_t));
2257 2267 sdp = next;
2258 2268 }
2259 2269 }
2260 2270
2261 2271 if (mp->sdt_tab)
2262 2272 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2263 2273 if (mp->text)
2264 2274 vmem_free(text_arena, mp->text, mp->text_size);
2265 2275 if (mp->data)
2266 2276 vmem_free(data_arena, mp->data, mp->data_size);
2267 2277 if (mp->depends_on)
2268 2278 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2269 2279 if (mp->filename)
2270 2280 kobj_free(mp->filename, strlen(mp->filename)+1);
2271 2281
2272 2282 kobj_free((char *)mp, sizeof (*mp));
2273 2283 }
2274 2284
2275 2285 static int
2276 2286 get_progbits(struct module *mp, struct _buf *file)
2277 2287 {
2278 2288 struct proginfo *tp, *dp, *sdp;
2279 2289 Shdr *shp;
2280 2290 reloc_dest_t dest = NULL;
2281 2291 uintptr_t bits_ptr;
2282 2292 uintptr_t text = 0, data, textptr;
2283 2293 uint_t shn;
2284 2294 int err = -1;
2285 2295
2286 2296 tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2287 2297 dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2288 2298 sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2289 2299 /*
2290 2300 * loop through sections to find out how much space we need
2291 2301 * for text, data, (also bss that is already assigned)
2292 2302 */
2293 2303 if (get_progbits_size(mp, tp, dp, sdp) < 0)
2294 2304 goto done;
2295 2305
2296 2306 mp->text_size = tp->size;
2297 2307 mp->data_size = dp->size;
2298 2308
2299 2309 if (standalone) {
2300 2310 caddr_t limit = _data;
2301 2311
2302 2312 if (lg_pagesize && _text + lg_pagesize < limit)
2303 2313 limit = _text + lg_pagesize;
2304 2314
2305 2315 mp->text = kobj_segbrk(&_etext, mp->text_size,
2306 2316 tp->align, limit);
2307 2317 /*
2308 2318 * If we can't grow the text segment, try the
2309 2319 * data segment before failing.
2310 2320 */
2311 2321 if (mp->text == NULL) {
2312 2322 mp->text = kobj_segbrk(&_edata, mp->text_size,
2313 2323 tp->align, 0);
2314 2324 }
2315 2325
2316 2326 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2317 2327
2318 2328 if (mp->text == NULL || mp->data == NULL)
2319 2329 goto done;
2320 2330
2321 2331 } else {
2322 2332 if (text_arena == NULL)
2323 2333 kobj_vmem_init(&text_arena, &data_arena);
2324 2334
2325 2335 /*
2326 2336 * some architectures may want to load the module on a
2327 2337 * page that is currently read only. It may not be
2328 2338 * possible for those architectures to remap their page
2329 2339 * on the fly. So we provide a facility for them to hang
2330 2340 * a private hook where the memory they assign the module
2331 2341 * is not the actual place where the module loads.
2332 2342 *
2333 2343 * In this case there are two addresses that deal with the
2334 2344 * modload.
2335 2345 * 1) the final destination of the module
2336 2346 * 2) the address that is used to view the newly
2337 2347 * loaded module until all the relocations relative to 1
2338 2348 * above are completed.
2339 2349 *
2340 2350 * That is what dest is used for below.
2341 2351 */
2342 2352 mp->text_size += tp->align;
2343 2353 mp->data_size += dp->align;
2344 2354
2345 2355 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2346 2356
2347 2357 /*
2348 2358 * a remap is taking place. Align the text ptr relative
2349 2359 * to the secondary mapping. That is where the bits will
2350 2360 * be read in.
2351 2361 */
2352 2362 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2353 2363 mp->text, mp->text_size)) {
2354 2364 off_t off = (uintptr_t)mp->text & PAGEOFFSET;
2355 2365 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2356 2366 caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2357 2367 caddr_t orig = mp->text - off;
2358 2368 pgcnt_t pages = size / PAGESIZE;
2359 2369
2360 2370 dest = (reloc_dest_t)(map + off);
2361 2371 text = ALIGN((uintptr_t)dest, tp->align);
2362 2372
2363 2373 while (pages--) {
2364 2374 hat_devload(kas.a_hat, map, PAGESIZE,
2365 2375 hat_getpfnum(kas.a_hat, orig),
2366 2376 PROT_READ | PROT_WRITE | PROT_EXEC,
2367 2377 HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2368 2378 map += PAGESIZE;
2369 2379 orig += PAGESIZE;
2370 2380 }
2371 2381 /*
2372 2382 * Since we set up a non-cacheable mapping, we need
2373 2383 * to flush any old entries in the cache that might
2374 2384 * be left around from the read-only mapping.
2375 2385 */
2376 2386 dcache_flushall();
2377 2387 }
2378 2388 if (mp->data_size)
2379 2389 mp->data = vmem_alloc(data_arena, mp->data_size,
2380 2390 VM_SLEEP | VM_BESTFIT);
2381 2391 }
2382 2392 textptr = (uintptr_t)mp->text;
2383 2393 textptr = ALIGN(textptr, tp->align);
2384 2394 mp->destination = dest;
2385 2395
2386 2396 /*
2387 2397 * This is the case where a remap is not being done.
2388 2398 */
2389 2399 if (text == 0)
2390 2400 text = ALIGN((uintptr_t)mp->text, tp->align);
2391 2401 data = ALIGN((uintptr_t)mp->data, dp->align);
2392 2402
2393 2403 /* now loop though sections assigning addresses and loading the data */
2394 2404 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2395 2405 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2396 2406 if (!(shp->sh_flags & SHF_ALLOC))
2397 2407 continue;
2398 2408
2399 2409 if ((shp->sh_flags & SHF_WRITE) == 0)
2400 2410 bits_ptr = text;
2401 2411 else
2402 2412 bits_ptr = data;
2403 2413
2404 2414 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2405 2415
2406 2416 if (shp->sh_type == SHT_NOBITS) {
2407 2417 /*
2408 2418 * Zero bss.
2409 2419 */
2410 2420 bzero((caddr_t)bits_ptr, shp->sh_size);
2411 2421 shp->sh_type = SHT_PROGBITS;
2412 2422 } else {
2413 2423 if (kobj_read_file(file, (char *)bits_ptr,
2414 2424 shp->sh_size, shp->sh_offset) < 0)
2415 2425 goto done;
2416 2426 }
2417 2427
2418 2428 if (shp->sh_flags & SHF_WRITE) {
2419 2429 shp->sh_addr = bits_ptr;
2420 2430 } else {
2421 2431 textptr = ALIGN(textptr, shp->sh_addralign);
2422 2432 shp->sh_addr = textptr;
2423 2433 textptr += shp->sh_size;
2424 2434 }
2425 2435
2426 2436 bits_ptr += shp->sh_size;
2427 2437 if ((shp->sh_flags & SHF_WRITE) == 0)
2428 2438 text = bits_ptr;
2429 2439 else
2430 2440 data = bits_ptr;
2431 2441 }
2432 2442
2433 2443 err = 0;
2434 2444 done:
2435 2445 /*
2436 2446 * Free and mark as freed the section headers here so that
2437 2447 * free_module_data() does not have to worry about this buffer.
2438 2448 *
2439 2449 * This buffer is freed here because one of the possible reasons
2440 2450 * for error is a section with non-zero sh_addr and in that case
2441 2451 * free_module_data() would have no way of recognizing that this
2442 2452 * buffer was unallocated.
2443 2453 */
2444 2454 if (err != 0) {
2445 2455 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2446 2456 mp->shdrs = NULL;
2447 2457 }
2448 2458
2449 2459 (void) kobj_free(tp, sizeof (struct proginfo));
2450 2460 (void) kobj_free(dp, sizeof (struct proginfo));
2451 2461 (void) kobj_free(sdp, sizeof (struct proginfo));
2452 2462
2453 2463 return (err);
2454 2464 }
2455 2465
2456 2466 /*
2457 2467 * Go through suppress_sym_list to see if "multiply defined"
2458 2468 * warning of this symbol should be suppressed. Return 1 if
2459 2469 * warning should be suppressed, 0 otherwise.
2460 2470 */
2461 2471 static int
2462 2472 kobj_suppress_warning(char *symname)
2463 2473 {
2464 2474 int i;
2465 2475
2466 2476 for (i = 0; suppress_sym_list[i] != NULL; i++) {
2467 2477 if (strcmp(suppress_sym_list[i], symname) == 0)
2468 2478 return (1);
2469 2479 }
2470 2480
2471 2481 return (0);
2472 2482 }
2473 2483
2474 2484 static int
2475 2485 get_syms(struct module *mp, struct _buf *file)
2476 2486 {
2477 2487 uint_t shn;
2478 2488 Shdr *shp;
2479 2489 uint_t i;
2480 2490 Sym *sp, *ksp;
2481 2491 char *symname;
2482 2492 int dosymtab = 0;
2483 2493
2484 2494 /*
2485 2495 * Find the interesting sections.
2486 2496 */
2487 2497 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2488 2498 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2489 2499 switch (shp->sh_type) {
2490 2500 case SHT_SYMTAB:
2491 2501 mp->symtbl_section = shn;
2492 2502 mp->symhdr = shp;
2493 2503 dosymtab++;
2494 2504 break;
2495 2505
2496 2506 case SHT_RELA:
2497 2507 case SHT_REL:
2498 2508 /*
2499 2509 * Already loaded.
2500 2510 */
2501 2511 if (shp->sh_addr)
2502 2512 continue;
2503 2513
2504 2514 /* KM_TMP since kobj_free'd in do_relocations */
2505 2515 shp->sh_addr = (Addr)
2506 2516 kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2507 2517
2508 2518 if (kobj_read_file(file, (char *)shp->sh_addr,
2509 2519 shp->sh_size, shp->sh_offset) < 0) {
2510 2520 _kobj_printf(ops, "krtld: get_syms: %s, ",
2511 2521 mp->filename);
2512 2522 _kobj_printf(ops, "error reading section %d\n",
2513 2523 shn);
2514 2524 return (-1);
2515 2525 }
2516 2526 break;
2517 2527 }
2518 2528 }
2519 2529
2520 2530 /*
2521 2531 * This is true for a stripped executable. In the case of
2522 2532 * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2523 2533 * and since that symbol information is still present everything
2524 2534 * is just fine.
2525 2535 */
2526 2536 if (!dosymtab) {
2527 2537 if (mp->flags & KOBJ_EXEC)
2528 2538 return (0);
2529 2539 _kobj_printf(ops, "krtld: get_syms: %s ",
2530 2540 mp->filename);
2531 2541 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2532 2542 return (-1);
2533 2543 }
2534 2544
2535 2545 /*
2536 2546 * get the associated string table header
2537 2547 */
2538 2548 if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2539 2549 return (-1);
2540 2550 mp->strhdr = (Shdr *)
2541 2551 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2542 2552
2543 2553 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2544 2554 mp->hashsize = kobj_gethashsize(mp->nsyms);
2545 2555
2546 2556 /*
2547 2557 * Allocate space for the symbol table, buckets, chains, and strings.
2548 2558 */
2549 2559 mp->symsize = mp->symhdr->sh_size +
2550 2560 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2551 2561 mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2552 2562
2553 2563 mp->symtbl = mp->symspace;
2554 2564 mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2555 2565 mp->chains = mp->buckets + mp->hashsize;
2556 2566 mp->strings = (char *)(mp->chains + mp->nsyms);
2557 2567
2558 2568 if (kobj_read_file(file, mp->symtbl,
2559 2569 mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2560 2570 kobj_read_file(file, mp->strings,
2561 2571 mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2562 2572 return (-1);
2563 2573
2564 2574 /*
2565 2575 * loop through the symbol table adjusting values to account
2566 2576 * for where each section got loaded into memory. Also
2567 2577 * fill in the hash table.
2568 2578 */
2569 2579 for (i = 1; i < mp->nsyms; i++) {
2570 2580 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2571 2581 if (sp->st_shndx < SHN_LORESERVE) {
2572 2582 if (sp->st_shndx >= mp->hdr.e_shnum) {
2573 2583 _kobj_printf(ops, "%s bad shndx ",
2574 2584 file->_name);
2575 2585 _kobj_printf(ops, "in symbol %d\n", i);
2576 2586 return (-1);
2577 2587 }
2578 2588 shp = (Shdr *)
2579 2589 (mp->shdrs +
2580 2590 sp->st_shndx * mp->hdr.e_shentsize);
2581 2591 if (!(mp->flags & KOBJ_EXEC))
2582 2592 sp->st_value += shp->sh_addr;
2583 2593 }
2584 2594
2585 2595 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2586 2596 continue;
2587 2597 if (sp->st_name >= mp->strhdr->sh_size)
2588 2598 return (-1);
2589 2599
2590 2600 symname = mp->strings + sp->st_name;
2591 2601
2592 2602 if (!(mp->flags & KOBJ_EXEC) &&
2593 2603 ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2594 2604 ksp = kobj_lookup_all(mp, symname, 0);
2595 2605
2596 2606 if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2597 2607 !kobj_suppress_warning(symname) &&
2598 2608 sp->st_shndx != SHN_UNDEF &&
2599 2609 sp->st_shndx != SHN_COMMON &&
2600 2610 ksp->st_shndx != SHN_UNDEF &&
2601 2611 ksp->st_shndx != SHN_COMMON) {
2602 2612 /*
2603 2613 * Unless this symbol is a stub, it's multiply
2604 2614 * defined. Multiply-defined symbols are
2605 2615 * usually bad, but some objects (kmdb) have
2606 2616 * a legitimate need to have their own
2607 2617 * copies of common functions.
2608 2618 */
2609 2619 if ((standalone ||
2610 2620 ksp->st_value < (uintptr_t)stubs_base ||
2611 2621 ksp->st_value >= (uintptr_t)stubs_end) &&
2612 2622 !(mp->flags & KOBJ_IGNMULDEF)) {
2613 2623 _kobj_printf(ops,
2614 2624 "%s symbol ", file->_name);
2615 2625 _kobj_printf(ops,
2616 2626 "%s multiply defined\n", symname);
2617 2627 }
2618 2628 }
2619 2629 }
2620 2630
2621 2631 sym_insert(mp, symname, i);
2622 2632 }
2623 2633
2624 2634 return (0);
2625 2635 }
2626 2636
2627 2637 static int
2628 2638 get_ctf(struct module *mp, struct _buf *file)
2629 2639 {
2630 2640 char *shstrtab, *ctfdata;
2631 2641 size_t shstrlen;
2632 2642 Shdr *shp;
2633 2643 uint_t i;
2634 2644
2635 2645 if (_moddebug & MODDEBUG_NOCTF)
2636 2646 return (0); /* do not attempt to even load CTF data */
2637 2647
2638 2648 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2639 2649 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2640 2650 mp->filename);
2641 2651 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2642 2652 mp->hdr.e_shstrndx);
2643 2653 return (-1);
2644 2654 }
2645 2655
2646 2656 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2647 2657 shstrlen = shp->sh_size;
2648 2658 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2649 2659
2650 2660 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2651 2661 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2652 2662 mp->filename);
2653 2663 _kobj_printf(ops, "error reading section %u\n",
2654 2664 mp->hdr.e_shstrndx);
2655 2665 kobj_free(shstrtab, shstrlen);
2656 2666 return (-1);
2657 2667 }
2658 2668
2659 2669 for (i = 0; i < mp->hdr.e_shnum; i++) {
2660 2670 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2661 2671
2662 2672 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2663 2673 strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2664 2674 ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2665 2675
2666 2676 if (kobj_read_file(file, ctfdata, shp->sh_size,
2667 2677 shp->sh_offset) < 0) {
2668 2678 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2669 2679 "reading .SUNW_ctf data\n", mp->filename);
2670 2680 kobj_free(ctfdata, shp->sh_size);
2671 2681 kobj_free(shstrtab, shstrlen);
2672 2682 return (-1);
2673 2683 }
2674 2684
2675 2685 mp->ctfdata = ctfdata;
2676 2686 mp->ctfsize = shp->sh_size;
2677 2687 break;
2678 2688 }
2679 2689 }
2680 2690
2681 2691 kobj_free(shstrtab, shstrlen);
2682 2692 return (0);
2683 2693 }
2684 2694
2685 2695 #define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */
2686 2696
2687 2697 /*
2688 2698 * Return the hash of the ELF sections that are memory resident.
2689 2699 * i.e. text and data. We skip a SHT_NOBITS section since it occupies
2690 2700 * no space in the file. We use SHA1 here since libelfsign uses
2691 2701 * it and both places need to use the same algorithm.
2692 2702 */
2693 2703 static void
2694 2704 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2695 2705 {
2696 2706 uint_t shn;
2697 2707 Shdr *shp;
2698 2708 SHA1_CTX ctx;
2699 2709
2700 2710 SHA1Init(&ctx);
2701 2711
2702 2712 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2703 2713 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2704 2714 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2705 2715 continue;
2706 2716
2707 2717 /*
2708 2718 * The check should ideally be shp->sh_type == SHT_NOBITS.
2709 2719 * However, we can't do that check here as get_progbits()
2710 2720 * resets the type.
2711 2721 */
2712 2722 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2713 2723 continue;
2714 2724 #ifdef KOBJ_DEBUG
2715 2725 if (kobj_debug & D_DEBUG)
2716 2726 _kobj_printf(ops,
2717 2727 "krtld: crypto_es_hash: updating hash with"
2718 2728 " %s data size=%d\n", shstrtab + shp->sh_name,
2719 2729 shp->sh_size);
2720 2730 #endif
2721 2731 ASSERT(shp->sh_addr != NULL);
2722 2732 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2723 2733 }
2724 2734
2725 2735 SHA1Final((uchar_t *)hash, &ctx);
2726 2736 }
2727 2737
2728 2738 /*
2729 2739 * Get the .SUNW_signature section for the module, it it exists.
2730 2740 *
2731 2741 * This section exists only for crypto modules. None of the
2732 2742 * primary modules have this section currently.
2733 2743 */
2734 2744 static void
2735 2745 get_signature(struct module *mp, struct _buf *file)
2736 2746 {
2737 2747 char *shstrtab, *sigdata = NULL;
2738 2748 size_t shstrlen;
2739 2749 Shdr *shp;
2740 2750 uint_t i;
2741 2751
2742 2752 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2743 2753 _kobj_printf(ops, "krtld: get_signature: %s, ",
2744 2754 mp->filename);
2745 2755 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2746 2756 mp->hdr.e_shstrndx);
2747 2757 return;
2748 2758 }
2749 2759
2750 2760 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2751 2761 shstrlen = shp->sh_size;
2752 2762 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2753 2763
2754 2764 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2755 2765 _kobj_printf(ops, "krtld: get_signature: %s, ",
2756 2766 mp->filename);
2757 2767 _kobj_printf(ops, "error reading section %u\n",
2758 2768 mp->hdr.e_shstrndx);
2759 2769 kobj_free(shstrtab, shstrlen);
2760 2770 return;
2761 2771 }
2762 2772
2763 2773 for (i = 0; i < mp->hdr.e_shnum; i++) {
2764 2774 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2765 2775 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2766 2776 strcmp(shstrtab + shp->sh_name,
2767 2777 ELF_SIGNATURE_SECTION) == 0) {
2768 2778 filesig_vers_t filesig_version;
2769 2779 size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2770 2780 sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2771 2781
2772 2782 if (kobj_read_file(file, sigdata, shp->sh_size,
2773 2783 shp->sh_offset) < 0) {
2774 2784 _kobj_printf(ops, "krtld: get_signature: %s,"
2775 2785 " error reading .SUNW_signature data\n",
2776 2786 mp->filename);
2777 2787 kobj_free(sigdata, sigsize);
2778 2788 kobj_free(shstrtab, shstrlen);
2779 2789 return;
2780 2790 }
2781 2791 filesig_version = ((struct filesignatures *)sigdata)->
2782 2792 filesig_sig.filesig_version;
2783 2793 if (!(filesig_version == FILESIG_VERSION1 ||
2784 2794 filesig_version == FILESIG_VERSION3)) {
2785 2795 /* skip versions we don't understand */
2786 2796 kobj_free(sigdata, sigsize);
2787 2797 kobj_free(shstrtab, shstrlen);
2788 2798 return;
2789 2799 }
2790 2800
2791 2801 mp->sigdata = sigdata;
2792 2802 mp->sigsize = sigsize;
2793 2803 break;
2794 2804 }
2795 2805 }
2796 2806
2797 2807 if (sigdata != NULL) {
2798 2808 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2799 2809 }
2800 2810
2801 2811 kobj_free(shstrtab, shstrlen);
2802 2812 }
2803 2813
2804 2814 static void
2805 2815 add_dependent(struct module *mp, struct module *dep)
2806 2816 {
2807 2817 struct module_list *lp;
2808 2818
2809 2819 for (lp = mp->head; lp; lp = lp->next) {
2810 2820 if (lp->mp == dep)
2811 2821 return; /* already on the list */
2812 2822 }
2813 2823
2814 2824 if (lp == NULL) {
2815 2825 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2816 2826
2817 2827 lp->mp = dep;
2818 2828 lp->next = NULL;
2819 2829 if (mp->tail)
2820 2830 mp->tail->next = lp;
2821 2831 else
2822 2832 mp->head = lp;
2823 2833 mp->tail = lp;
2824 2834 }
2825 2835 }
2826 2836
2827 2837 static int
2828 2838 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2829 2839 {
2830 2840 struct module *mp;
2831 2841 struct modctl *req;
2832 2842 char *d, *p, *q;
2833 2843 int c;
2834 2844 char *err_modname = NULL;
2835 2845
2836 2846 mp = modp->mod_mp;
2837 2847
2838 2848 if ((p = mp->depends_on) == NULL)
2839 2849 return (0);
2840 2850
2841 2851 for (;;) {
2842 2852 /*
2843 2853 * Skip space.
2844 2854 */
2845 2855 while (*p && (*p == ' ' || *p == '\t'))
2846 2856 p++;
2847 2857 /*
2848 2858 * Get module name.
2849 2859 */
2850 2860 d = p;
2851 2861 q = modname;
2852 2862 c = 0;
2853 2863 while (*p && *p != ' ' && *p != '\t') {
2854 2864 if (c < modnamelen - 1) {
2855 2865 *q++ = *p;
2856 2866 c++;
2857 2867 }
2858 2868 p++;
2859 2869 }
2860 2870
2861 2871 if (q == modname)
2862 2872 break;
2863 2873
2864 2874 if (c == modnamelen - 1) {
2865 2875 char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2866 2876
2867 2877 (void) strncpy(dep, d, p - d + 1);
2868 2878 dep[p - d] = '\0';
2869 2879
2870 2880 _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2871 2881 _kobj_printf(ops, "'%s' too long ", dep);
2872 2882 _kobj_printf(ops, "(max %d chars)\n", modnamelen);
2873 2883
2874 2884 kobj_free(dep, p - d + 1);
2875 2885
2876 2886 return (-1);
2877 2887 }
2878 2888
2879 2889 *q = '\0';
2880 2890 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2881 2891 #ifndef KOBJ_DEBUG
2882 2892 if (_moddebug & MODDEBUG_LOADMSG) {
2883 2893 #endif /* KOBJ_DEBUG */
2884 2894 _kobj_printf(ops,
2885 2895 "%s: unable to resolve dependency, ",
2886 2896 modp->mod_modname);
2887 2897 _kobj_printf(ops, "cannot load module '%s'\n",
2888 2898 modname);
2889 2899 #ifndef KOBJ_DEBUG
2890 2900 }
2891 2901 #endif /* KOBJ_DEBUG */
2892 2902 if (err_modname == NULL) {
2893 2903 /*
2894 2904 * This must be the same size as the modname
2895 2905 * one.
2896 2906 */
2897 2907 err_modname = kobj_zalloc(MODMAXNAMELEN,
2898 2908 KM_WAIT);
2899 2909
2900 2910 /*
2901 2911 * We can use strcpy() here without fearing
2902 2912 * the NULL terminator because the size of
2903 2913 * err_modname is the same as one of modname,
2904 2914 * and it's filled with zeros.
2905 2915 */
2906 2916 (void) strcpy(err_modname, modname);
2907 2917 }
2908 2918 continue;
2909 2919 }
2910 2920
2911 2921 add_dependent(mp, req->mod_mp);
2912 2922 mod_release_mod(req);
2913 2923
2914 2924 }
2915 2925
2916 2926 if (err_modname != NULL) {
2917 2927 /*
2918 2928 * Copy the first module name where you detect an error to keep
2919 2929 * its behavior the same as before.
2920 2930 * This way keeps minimizing the memory use for error
2921 2931 * modules, and this might be important at boot time because
2922 2932 * the memory usage is a crucial factor for booting in most
2923 2933 * cases. You can expect more verbose messages when using
2924 2934 * a debug kernel or setting a bit in moddebug.
2925 2935 */
2926 2936 bzero(modname, MODMAXNAMELEN);
2927 2937 (void) strcpy(modname, err_modname);
2928 2938 kobj_free(err_modname, MODMAXNAMELEN);
2929 2939 return (-1);
2930 2940 }
2931 2941
2932 2942 return (0);
2933 2943 }
2934 2944
2935 2945 static int
2936 2946 do_common(struct module *mp)
2937 2947 {
2938 2948 int err;
2939 2949
2940 2950 /*
2941 2951 * first time through, assign all symbols defined in other
2942 2952 * modules, and count up how much common space will be needed
2943 2953 * (bss_size and bss_align)
2944 2954 */
2945 2955 if ((err = do_symbols(mp, 0)) < 0)
2946 2956 return (err);
2947 2957 /*
2948 2958 * increase bss_size by the maximum delta that could be
2949 2959 * computed by the ALIGN below
2950 2960 */
2951 2961 mp->bss_size += mp->bss_align;
2952 2962 if (mp->bss_size) {
2953 2963 if (standalone)
2954 2964 mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2955 2965 MINALIGN, 0);
2956 2966 else
2957 2967 mp->bss = (uintptr_t)vmem_alloc(data_arena,
2958 2968 mp->bss_size, VM_SLEEP | VM_BESTFIT);
2959 2969 bzero((void *)mp->bss, mp->bss_size);
2960 2970 /* now assign addresses to all common symbols */
2961 2971 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2962 2972 return (err);
2963 2973 }
2964 2974 return (0);
2965 2975 }
2966 2976
2967 2977 static int
2968 2978 do_symbols(struct module *mp, Elf64_Addr bss_base)
2969 2979 {
2970 2980 int bss_align;
2971 2981 uintptr_t bss_ptr;
2972 2982 int err;
2973 2983 int i;
2974 2984 Sym *sp, *sp1;
2975 2985 char *name;
2976 2986 int assign;
2977 2987 int resolved = 1;
2978 2988
2979 2989 /*
2980 2990 * Nothing left to do (optimization).
2981 2991 */
2982 2992 if (mp->flags & KOBJ_RESOLVED)
2983 2993 return (0);
2984 2994
2985 2995 assign = (bss_base) ? 1 : 0;
2986 2996 bss_ptr = bss_base;
2987 2997 bss_align = 0;
2988 2998 err = 0;
2989 2999
2990 3000 for (i = 1; i < mp->nsyms; i++) {
2991 3001 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
2992 3002 /*
2993 3003 * we know that st_name is in bounds, since get_sections
2994 3004 * has already checked all of the symbols
2995 3005 */
2996 3006 name = mp->strings + sp->st_name;
2997 3007 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
2998 3008 continue;
2999 3009 #if defined(__sparc)
3000 3010 /*
3001 3011 * Register symbols are ignored in the kernel
3002 3012 */
3003 3013 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER) {
3004 3014 if (*name != '\0') {
3005 3015 _kobj_printf(ops, "%s: named REGISTER symbol ",
3006 3016 mp->filename);
3007 3017 _kobj_printf(ops, "not supported '%s'\n",
3008 3018 name);
3009 3019 err = DOSYM_UNDEF;
3010 3020 }
3011 3021 continue;
3012 3022 }
3013 3023 #endif /* __sparc */
3014 3024 /*
3015 3025 * TLS symbols are ignored in the kernel
3016 3026 */
3017 3027 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
3018 3028 _kobj_printf(ops, "%s: TLS symbol ",
3019 3029 mp->filename);
3020 3030 _kobj_printf(ops, "not supported '%s'\n",
3021 3031 name);
3022 3032 err = DOSYM_UNDEF;
3023 3033 continue;
3024 3034 }
3025 3035
3026 3036 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
|
↓ open down ↓ |
825 lines elided |
↑ open up ↑ |
3027 3037 if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
3028 3038 sp->st_shndx = SHN_ABS;
3029 3039 sp->st_value = sp1->st_value;
3030 3040 continue;
3031 3041 }
3032 3042 }
3033 3043
3034 3044 if (sp->st_shndx == SHN_UNDEF) {
3035 3045 resolved = 0;
3036 3046
3047 + /*
3048 + * Skip over sdt probes and smap calls,
3049 + * they're relocated later.
3050 + */
3037 3051 if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
3038 3052 continue;
3053 +#if defined(__x86)
3054 + if (strcmp(name, "smap_enable") == 0 ||
3055 + strcmp(name, "smap_disable") == 0)
3056 + continue;
3057 +#endif /* defined(__x86) */
3039 3058
3059 +
3040 3060 /*
3041 3061 * If it's not a weak reference and it's
3042 3062 * not a primary object, it's an error.
3043 3063 * (Primary objects may take more than
3044 3064 * one pass to resolve)
3045 3065 */
3046 3066 if (!(mp->flags & KOBJ_PRIM) &&
3047 3067 ELF_ST_BIND(sp->st_info) != STB_WEAK) {
3048 3068 _kobj_printf(ops, "%s: undefined symbol",
3049 3069 mp->filename);
3050 3070 _kobj_printf(ops, " '%s'\n", name);
3051 3071 /*
3052 3072 * Try to determine whether this symbol
3053 3073 * represents a dependency on obsolete
3054 3074 * unsafe driver support. This is just
3055 3075 * to make the warning more informative.
3056 3076 */
3057 3077 if (strcmp(name, "sleep") == 0 ||
3058 3078 strcmp(name, "unsleep") == 0 ||
3059 3079 strcmp(name, "wakeup") == 0 ||
3060 3080 strcmp(name, "bsd_compat_ioctl") == 0 ||
3061 3081 strcmp(name, "unsafe_driver") == 0 ||
3062 3082 strncmp(name, "spl", 3) == 0 ||
3063 3083 strncmp(name, "i_ddi_spl", 9) == 0)
3064 3084 err = DOSYM_UNSAFE;
3065 3085 if (err == 0)
3066 3086 err = DOSYM_UNDEF;
3067 3087 }
3068 3088 continue;
3069 3089 }
3070 3090 /*
3071 3091 * It's a common symbol - st_value is the
3072 3092 * required alignment.
3073 3093 */
3074 3094 if (sp->st_value > bss_align)
3075 3095 bss_align = sp->st_value;
3076 3096 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3077 3097 if (assign) {
3078 3098 sp->st_shndx = SHN_ABS;
3079 3099 sp->st_value = bss_ptr;
3080 3100 }
3081 3101 bss_ptr += sp->st_size;
3082 3102 }
3083 3103 if (err)
3084 3104 return (err);
3085 3105 if (assign == 0 && mp->bss == 0) {
3086 3106 mp->bss_align = bss_align;
3087 3107 mp->bss_size = bss_ptr;
3088 3108 } else if (resolved) {
3089 3109 mp->flags |= KOBJ_RESOLVED;
3090 3110 }
3091 3111
3092 3112 return (0);
3093 3113 }
3094 3114
3095 3115 uint_t
3096 3116 kobj_hash_name(const char *p)
3097 3117 {
3098 3118 uint_t g;
3099 3119 uint_t hval;
3100 3120
3101 3121 hval = 0;
3102 3122 while (*p) {
3103 3123 hval = (hval << 4) + *p++;
3104 3124 if ((g = (hval & 0xf0000000)) != 0)
3105 3125 hval ^= g >> 24;
3106 3126 hval &= ~g;
3107 3127 }
3108 3128 return (hval);
3109 3129 }
3110 3130
3111 3131 /* look for name in all modules */
3112 3132 uintptr_t
3113 3133 kobj_getsymvalue(char *name, int kernelonly)
3114 3134 {
3115 3135 Sym *sp;
3116 3136 struct modctl *modp;
3117 3137 struct module *mp;
3118 3138 uintptr_t value = 0;
3119 3139
3120 3140 if ((sp = kobj_lookup_kernel(name)) != NULL)
3121 3141 return ((uintptr_t)sp->st_value);
3122 3142
3123 3143 if (kernelonly)
3124 3144 return (0); /* didn't find it in the kernel so give up */
3125 3145
3126 3146 mutex_enter(&mod_lock);
3127 3147 modp = &modules;
3128 3148 do {
3129 3149 mp = (struct module *)modp->mod_mp;
3130 3150 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3131 3151 (sp = lookup_one(mp, name))) {
3132 3152 value = (uintptr_t)sp->st_value;
3133 3153 break;
3134 3154 }
3135 3155 } while ((modp = modp->mod_next) != &modules);
3136 3156 mutex_exit(&mod_lock);
3137 3157 return (value);
3138 3158 }
3139 3159
3140 3160 /* look for a symbol near value. */
3141 3161 char *
3142 3162 kobj_getsymname(uintptr_t value, ulong_t *offset)
3143 3163 {
3144 3164 char *name = NULL;
3145 3165 struct modctl *modp;
3146 3166
3147 3167 struct modctl_list *lp;
3148 3168 struct module *mp;
3149 3169
3150 3170 /*
3151 3171 * Loop through the primary kernel modules.
3152 3172 */
3153 3173 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3154 3174 mp = mod(lp);
3155 3175
3156 3176 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3157 3177 return (name);
3158 3178 }
3159 3179
3160 3180 mutex_enter(&mod_lock);
3161 3181 modp = &modules;
3162 3182 do {
3163 3183 mp = (struct module *)modp->mod_mp;
3164 3184 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3165 3185 (name = kobj_searchsym(mp, value, offset)))
3166 3186 break;
3167 3187 } while ((modp = modp->mod_next) != &modules);
3168 3188 mutex_exit(&mod_lock);
3169 3189 return (name);
3170 3190 }
3171 3191
3172 3192 /* return address of symbol and size */
3173 3193
3174 3194 uintptr_t
3175 3195 kobj_getelfsym(char *name, void *mp, int *size)
3176 3196 {
3177 3197 Sym *sp;
3178 3198
3179 3199 if (mp == NULL)
3180 3200 sp = kobj_lookup_kernel(name);
3181 3201 else
3182 3202 sp = lookup_one(mp, name);
3183 3203
3184 3204 if (sp == NULL)
3185 3205 return (0);
3186 3206
3187 3207 *size = (int)sp->st_size;
3188 3208 return ((uintptr_t)sp->st_value);
3189 3209 }
3190 3210
3191 3211 uintptr_t
3192 3212 kobj_lookup(struct module *mod, const char *name)
3193 3213 {
3194 3214 Sym *sp;
3195 3215
3196 3216 sp = lookup_one(mod, name);
3197 3217
3198 3218 if (sp == NULL)
3199 3219 return (0);
3200 3220
3201 3221 return ((uintptr_t)sp->st_value);
3202 3222 }
3203 3223
3204 3224 char *
3205 3225 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3206 3226 {
3207 3227 Sym *symtabptr;
3208 3228 char *strtabptr;
3209 3229 int symnum;
3210 3230 Sym *sym;
3211 3231 Sym *cursym;
3212 3232 uintptr_t curval;
3213 3233
3214 3234 *offset = (ulong_t)-1l; /* assume not found */
3215 3235 cursym = NULL;
3216 3236
3217 3237 if (kobj_addrcheck(mp, (void *)value) != 0)
3218 3238 return (NULL); /* not in this module */
3219 3239
3220 3240 strtabptr = mp->strings;
3221 3241 symtabptr = (Sym *)mp->symtbl;
3222 3242
3223 3243 /*
3224 3244 * Scan the module's symbol table for a symbol <= value
3225 3245 */
3226 3246 for (symnum = 1, sym = symtabptr + 1;
3227 3247 symnum < mp->nsyms; symnum++, sym = (Sym *)
3228 3248 ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3229 3249 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3230 3250 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3231 3251 continue;
3232 3252 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3233 3253 ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3234 3254 continue;
3235 3255 }
3236 3256
3237 3257 curval = (uintptr_t)sym->st_value;
3238 3258
3239 3259 if (curval > value)
3240 3260 continue;
3241 3261
3242 3262 /*
3243 3263 * If one or both are functions...
3244 3264 */
3245 3265 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3246 3266 ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3247 3267 /* Ignore if the address is out of the bounds */
3248 3268 if (value - sym->st_value >= sym->st_size)
3249 3269 continue;
3250 3270
3251 3271 if (cursym != NULL &&
3252 3272 ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3253 3273 /* Prefer the function to the non-function */
3254 3274 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3255 3275 continue;
3256 3276
3257 3277 /* Prefer the larger of the two functions */
3258 3278 if (sym->st_size <= cursym->st_size)
3259 3279 continue;
3260 3280 }
3261 3281 } else if (value - curval >= *offset) {
3262 3282 continue;
3263 3283 }
3264 3284
3265 3285 *offset = (ulong_t)(value - curval);
3266 3286 cursym = sym;
3267 3287 }
3268 3288 if (cursym == NULL)
3269 3289 return (NULL);
3270 3290
3271 3291 return (strtabptr + cursym->st_name);
3272 3292 }
3273 3293
3274 3294 Sym *
3275 3295 kobj_lookup_all(struct module *mp, char *name, int include_self)
3276 3296 {
3277 3297 Sym *sp;
3278 3298 struct module_list *mlp;
3279 3299 struct modctl_list *clp;
3280 3300 struct module *mmp;
3281 3301
3282 3302 if (include_self && (sp = lookup_one(mp, name)) != NULL)
3283 3303 return (sp);
3284 3304
3285 3305 for (mlp = mp->head; mlp; mlp = mlp->next) {
3286 3306 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3287 3307 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3288 3308 return (sp);
3289 3309 }
3290 3310
3291 3311 /*
3292 3312 * Loop through the primary kernel modules.
3293 3313 */
3294 3314 for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3295 3315 mmp = mod(clp);
3296 3316
3297 3317 if (mmp == NULL || mp == mmp)
3298 3318 continue;
3299 3319
3300 3320 if ((sp = lookup_one(mmp, name)) != NULL &&
3301 3321 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3302 3322 return (sp);
3303 3323 }
3304 3324 return (NULL);
3305 3325 }
3306 3326
3307 3327 Sym *
3308 3328 kobj_lookup_kernel(const char *name)
3309 3329 {
3310 3330 struct modctl_list *lp;
3311 3331 struct module *mp;
3312 3332 Sym *sp;
3313 3333
3314 3334 /*
3315 3335 * Loop through the primary kernel modules.
3316 3336 */
3317 3337 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3318 3338 mp = mod(lp);
3319 3339
3320 3340 if (mp == NULL)
3321 3341 continue;
3322 3342
3323 3343 if ((sp = lookup_one(mp, name)) != NULL)
3324 3344 return (sp);
3325 3345 }
3326 3346 return (NULL);
3327 3347 }
3328 3348
3329 3349 static Sym *
3330 3350 lookup_one(struct module *mp, const char *name)
3331 3351 {
3332 3352 symid_t *ip;
3333 3353 char *name1;
3334 3354 Sym *sp;
3335 3355
3336 3356 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3337 3357 ip = &mp->chains[*ip]) {
3338 3358 sp = (Sym *)(mp->symtbl +
3339 3359 mp->symhdr->sh_entsize * *ip);
3340 3360 name1 = mp->strings + sp->st_name;
3341 3361 if (strcmp(name, name1) == 0 &&
3342 3362 ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3343 3363 sp->st_shndx != SHN_UNDEF &&
3344 3364 sp->st_shndx != SHN_COMMON)
3345 3365 return (sp);
3346 3366 }
3347 3367 return (NULL);
3348 3368 }
3349 3369
3350 3370 /*
3351 3371 * Lookup a given symbol pointer in the module's symbol hash. If the symbol
3352 3372 * is hashed, return the symbol pointer; otherwise return NULL.
3353 3373 */
3354 3374 static Sym *
3355 3375 sym_lookup(struct module *mp, Sym *ksp)
3356 3376 {
3357 3377 char *name = mp->strings + ksp->st_name;
3358 3378 symid_t *ip;
3359 3379 Sym *sp;
3360 3380
3361 3381 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3362 3382 ip = &mp->chains[*ip]) {
3363 3383 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3364 3384 if (sp == ksp)
3365 3385 return (ksp);
3366 3386 }
3367 3387 return (NULL);
3368 3388 }
3369 3389
3370 3390 static void
3371 3391 sym_insert(struct module *mp, char *name, symid_t index)
3372 3392 {
3373 3393 symid_t *ip;
3374 3394
3375 3395 #ifdef KOBJ_DEBUG
3376 3396 if (kobj_debug & D_SYMBOLS) {
3377 3397 static struct module *lastmp = NULL;
3378 3398 Sym *sp;
3379 3399 if (lastmp != mp) {
3380 3400 _kobj_printf(ops,
3381 3401 "krtld: symbol entry: file=%s\n",
3382 3402 mp->filename);
3383 3403 _kobj_printf(ops,
3384 3404 "krtld:\tsymndx\tvalue\t\t"
3385 3405 "symbol name\n");
3386 3406 lastmp = mp;
3387 3407 }
3388 3408 sp = (Sym *)(mp->symtbl +
3389 3409 index * mp->symhdr->sh_entsize);
3390 3410 _kobj_printf(ops, "krtld:\t[%3d]", index);
3391 3411 _kobj_printf(ops, "\t0x%lx", sp->st_value);
3392 3412 _kobj_printf(ops, "\t%s\n", name);
3393 3413 }
3394 3414 #endif
3395 3415
3396 3416 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3397 3417 ip = &mp->chains[*ip]) {
3398 3418 ;
3399 3419 }
3400 3420 *ip = index;
3401 3421 }
3402 3422
3403 3423 struct modctl *
3404 3424 kobj_boot_mod_lookup(const char *modname)
3405 3425 {
3406 3426 struct modctl *mctl = kobj_modules;
3407 3427
3408 3428 do {
3409 3429 if (strcmp(modname, mctl->mod_modname) == 0)
3410 3430 return (mctl);
3411 3431 } while ((mctl = mctl->mod_next) != kobj_modules);
3412 3432
3413 3433 return (NULL);
3414 3434 }
3415 3435
3416 3436 /*
3417 3437 * Determine if the module exists.
3418 3438 */
3419 3439 int
3420 3440 kobj_path_exists(char *name, int use_path)
3421 3441 {
3422 3442 struct _buf *file;
3423 3443
3424 3444 file = kobj_open_path(name, use_path, 1);
3425 3445 #ifdef MODDIR_SUFFIX
3426 3446 if (file == (struct _buf *)-1)
3427 3447 file = kobj_open_path(name, use_path, 0);
3428 3448 #endif /* MODDIR_SUFFIX */
3429 3449 if (file == (struct _buf *)-1)
3430 3450 return (0);
3431 3451 kobj_close_file(file);
3432 3452 return (1);
3433 3453 }
3434 3454
3435 3455 /*
3436 3456 * fullname is dynamically allocated to be able to hold the
3437 3457 * maximum size string that can be constructed from name.
3438 3458 * path is exactly like the shell PATH variable.
3439 3459 */
3440 3460 struct _buf *
3441 3461 kobj_open_path(char *name, int use_path, int use_moddir_suffix)
3442 3462 {
3443 3463 char *p, *q;
3444 3464 char *pathp;
3445 3465 char *pathpsave;
3446 3466 char *fullname;
3447 3467 int maxpathlen;
3448 3468 struct _buf *file;
3449 3469
3450 3470 #if !defined(MODDIR_SUFFIX)
3451 3471 use_moddir_suffix = B_FALSE;
3452 3472 #endif
3453 3473
3454 3474 if (!use_path)
3455 3475 pathp = ""; /* use name as specified */
3456 3476 else
3457 3477 pathp = kobj_module_path;
3458 3478 /* use configured default path */
3459 3479
3460 3480 pathpsave = pathp; /* keep this for error reporting */
3461 3481
3462 3482 /*
3463 3483 * Allocate enough space for the largest possible fullname.
3464 3484 * since path is of the form <directory> : <directory> : ...
3465 3485 * we're potentially allocating a little more than we need to
3466 3486 * but we'll allocate the exact amount when we find the right directory.
3467 3487 * (The + 3 below is one for NULL terminator and one for the '/'
3468 3488 * we might have to add at the beginning of path and one for
3469 3489 * the '/' between path and name.)
3470 3490 */
3471 3491 maxpathlen = strlen(pathp) + strlen(name) + 3;
3472 3492 /* sizeof includes null */
3473 3493 maxpathlen += sizeof (slash_moddir_suffix_slash) - 1;
3474 3494 fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3475 3495
3476 3496 for (;;) {
3477 3497 p = fullname;
3478 3498 if (*pathp != '\0' && *pathp != '/')
3479 3499 *p++ = '/'; /* path must start with '/' */
3480 3500 while (*pathp && *pathp != ':' && *pathp != ' ')
3481 3501 *p++ = *pathp++;
3482 3502 if (p != fullname && p[-1] != '/')
3483 3503 *p++ = '/';
3484 3504 if (use_moddir_suffix) {
3485 3505 char *b = basename(name);
3486 3506 char *s;
3487 3507
3488 3508 /* copy everything up to the base name */
3489 3509 q = name;
3490 3510 while (q != b && *q)
3491 3511 *p++ = *q++;
3492 3512 s = slash_moddir_suffix_slash;
3493 3513 while (*s)
3494 3514 *p++ = *s++;
3495 3515 /* copy the rest */
3496 3516 while (*b)
3497 3517 *p++ = *b++;
3498 3518 } else {
3499 3519 q = name;
3500 3520 while (*q)
3501 3521 *p++ = *q++;
3502 3522 }
3503 3523 *p = 0;
3504 3524 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3505 3525 kobj_free(fullname, maxpathlen);
3506 3526 return (file);
3507 3527 }
3508 3528 while (*pathp == ' ' || *pathp == ':')
3509 3529 pathp++;
3510 3530 if (*pathp == 0)
3511 3531 break;
3512 3532
3513 3533 }
3514 3534 kobj_free(fullname, maxpathlen);
3515 3535 if (_moddebug & MODDEBUG_ERRMSG) {
3516 3536 _kobj_printf(ops, "can't open %s,", name);
3517 3537 _kobj_printf(ops, " path is %s\n", pathpsave);
3518 3538 }
3519 3539 return ((struct _buf *)-1);
3520 3540 }
3521 3541
3522 3542 intptr_t
3523 3543 kobj_open(char *filename)
3524 3544 {
3525 3545 struct vnode *vp;
3526 3546 int fd;
3527 3547
3528 3548 if (_modrootloaded) {
3529 3549 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3530 3550 int Errno;
3531 3551
3532 3552 /*
3533 3553 * Hand off the open to a thread who has a
3534 3554 * stack size capable handling the request.
3535 3555 */
3536 3556 if (curthread != &t0) {
3537 3557 (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3538 3558 kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3539 3559 sema_p(<p->sema);
3540 3560 Errno = ltp->Errno;
3541 3561 vp = ltp->vp;
3542 3562 } else {
3543 3563 /*
3544 3564 * 1098067: module creds should not be those of the
3545 3565 * caller
3546 3566 */
3547 3567 cred_t *saved_cred = curthread->t_cred;
3548 3568 curthread->t_cred = kcred;
3549 3569 Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3550 3570 0, 0, rootdir, -1);
3551 3571 curthread->t_cred = saved_cred;
3552 3572 }
3553 3573 kobjopen_free(ltp);
3554 3574
3555 3575 if (Errno) {
3556 3576 if (_moddebug & MODDEBUG_ERRMSG) {
3557 3577 _kobj_printf(ops,
3558 3578 "kobj_open: vn_open of %s fails, ",
3559 3579 filename);
3560 3580 _kobj_printf(ops, "Errno = %d\n", Errno);
3561 3581 }
3562 3582 return (-1);
3563 3583 } else {
3564 3584 if (_moddebug & MODDEBUG_ERRMSG) {
3565 3585 _kobj_printf(ops, "kobj_open: '%s'", filename);
3566 3586 _kobj_printf(ops, " vp = %p\n", vp);
3567 3587 }
3568 3588 return ((intptr_t)vp);
3569 3589 }
3570 3590 } else {
3571 3591 fd = kobj_boot_open(filename, 0);
3572 3592
3573 3593 if (_moddebug & MODDEBUG_ERRMSG) {
3574 3594 if (fd < 0)
3575 3595 _kobj_printf(ops,
3576 3596 "kobj_open: can't open %s\n", filename);
3577 3597 else {
3578 3598 _kobj_printf(ops, "kobj_open: '%s'", filename);
3579 3599 _kobj_printf(ops, " descr = 0x%x\n", fd);
3580 3600 }
3581 3601 }
3582 3602 return ((intptr_t)fd);
3583 3603 }
3584 3604 }
3585 3605
3586 3606 /*
3587 3607 * Calls to kobj_open() are handled off to this routine as a separate thread.
3588 3608 */
3589 3609 static void
3590 3610 kobjopen_thread(struct kobjopen_tctl *ltp)
3591 3611 {
3592 3612 kmutex_t cpr_lk;
3593 3613 callb_cpr_t cpr_i;
3594 3614
3595 3615 mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3596 3616 CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3597 3617 ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3598 3618 0, 0);
3599 3619 sema_v(<p->sema);
3600 3620 mutex_enter(&cpr_lk);
3601 3621 CALLB_CPR_EXIT(&cpr_i);
3602 3622 mutex_destroy(&cpr_lk);
3603 3623 thread_exit();
3604 3624 }
3605 3625
3606 3626 /*
3607 3627 * allocate and initialize a kobjopen thread structure
3608 3628 */
3609 3629 static struct kobjopen_tctl *
3610 3630 kobjopen_alloc(char *filename)
3611 3631 {
3612 3632 struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3613 3633
3614 3634 ASSERT(filename != NULL);
3615 3635
3616 3636 ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3617 3637 bcopy(filename, ltp->name, strlen(filename) + 1);
3618 3638 sema_init(<p->sema, 0, NULL, SEMA_DEFAULT, NULL);
3619 3639 return (ltp);
3620 3640 }
3621 3641
3622 3642 /*
3623 3643 * free a kobjopen thread control structure
3624 3644 */
3625 3645 static void
3626 3646 kobjopen_free(struct kobjopen_tctl *ltp)
3627 3647 {
3628 3648 sema_destroy(<p->sema);
3629 3649 kmem_free(ltp->name, strlen(ltp->name) + 1);
3630 3650 kmem_free(ltp, sizeof (*ltp));
3631 3651 }
3632 3652
3633 3653 int
3634 3654 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3635 3655 {
3636 3656 int stat;
3637 3657 ssize_t resid;
3638 3658
3639 3659 if (_modrootloaded) {
3640 3660 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3641 3661 (offset_t)offset, UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3642 3662 &resid)) != 0) {
3643 3663 _kobj_printf(ops,
3644 3664 "vn_rdwr failed with error 0x%x\n", stat);
3645 3665 return (-1);
3646 3666 }
3647 3667 return (size - resid);
3648 3668 } else {
3649 3669 int count = 0;
3650 3670
3651 3671 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3652 3672 _kobj_printf(ops,
3653 3673 "kobj_read: seek 0x%x failed\n", offset);
3654 3674 return (-1);
3655 3675 }
3656 3676
3657 3677 count = kobj_boot_read((int)descr, buf, size);
3658 3678 if (count < size) {
3659 3679 if (_moddebug & MODDEBUG_ERRMSG) {
3660 3680 _kobj_printf(ops,
3661 3681 "kobj_read: req %d bytes, ", size);
3662 3682 _kobj_printf(ops, "got %d\n", count);
3663 3683 }
3664 3684 }
3665 3685 return (count);
3666 3686 }
3667 3687 }
3668 3688
3669 3689 void
3670 3690 kobj_close(intptr_t descr)
3671 3691 {
3672 3692 if (_moddebug & MODDEBUG_ERRMSG)
3673 3693 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3674 3694
3675 3695 if (_modrootloaded) {
3676 3696 struct vnode *vp = (struct vnode *)descr;
3677 3697 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
3678 3698 VN_RELE(vp);
3679 3699 } else
3680 3700 (void) kobj_boot_close((int)descr);
3681 3701 }
3682 3702
3683 3703 int
3684 3704 kobj_fstat(intptr_t descr, struct bootstat *buf)
3685 3705 {
3686 3706 if (buf == NULL)
3687 3707 return (-1);
3688 3708
3689 3709 if (_modrootloaded) {
3690 3710 vattr_t vattr;
3691 3711 struct vnode *vp = (struct vnode *)descr;
3692 3712 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3693 3713 return (-1);
3694 3714
3695 3715 /*
3696 3716 * The vattr and bootstat structures are similar, but not
3697 3717 * identical. We do our best to fill in the bootstat structure
3698 3718 * from the contents of vattr (transfering only the ones that
3699 3719 * are obvious.
3700 3720 */
3701 3721
3702 3722 buf->st_mode = (uint32_t)vattr.va_mode;
3703 3723 buf->st_nlink = (uint32_t)vattr.va_nlink;
3704 3724 buf->st_uid = (int32_t)vattr.va_uid;
3705 3725 buf->st_gid = (int32_t)vattr.va_gid;
3706 3726 buf->st_rdev = (uint64_t)vattr.va_rdev;
3707 3727 buf->st_size = (uint64_t)vattr.va_size;
3708 3728 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3709 3729 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3710 3730 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3711 3731 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3712 3732 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3713 3733 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3714 3734 buf->st_blksize = (int32_t)vattr.va_blksize;
3715 3735 buf->st_blocks = (int64_t)vattr.va_nblocks;
3716 3736
3717 3737 return (0);
3718 3738 }
3719 3739
3720 3740 return (kobj_boot_fstat((int)descr, buf));
3721 3741 }
3722 3742
3723 3743
3724 3744 struct _buf *
3725 3745 kobj_open_file(char *name)
3726 3746 {
3727 3747 struct _buf *file;
3728 3748 struct compinfo cbuf;
3729 3749 intptr_t fd;
3730 3750
3731 3751 if ((fd = kobj_open(name)) == -1) {
3732 3752 return ((struct _buf *)-1);
3733 3753 }
3734 3754
3735 3755 file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3736 3756 file->_fd = fd;
3737 3757 file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3738 3758 file->_cnt = file->_size = file->_off = 0;
3739 3759 file->_ln = 1;
3740 3760 file->_ptr = file->_base;
3741 3761 (void) strcpy(file->_name, name);
3742 3762
3743 3763 /*
3744 3764 * Before root is mounted, we must check
3745 3765 * for a compressed file and do our own
3746 3766 * buffering.
3747 3767 */
3748 3768 if (_modrootloaded) {
3749 3769 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3750 3770 file->_bsize = MAXBSIZE;
3751 3771
3752 3772 /* Check if the file is compressed */
3753 3773 file->_iscmp = kobj_is_compressed(fd);
3754 3774 } else {
3755 3775 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3756 3776 kobj_close_file(file);
3757 3777 return ((struct _buf *)-1);
3758 3778 }
3759 3779 file->_iscmp = cbuf.iscmp;
3760 3780 if (file->_iscmp) {
3761 3781 if (kobj_comp_setup(file, &cbuf) != 0) {
3762 3782 kobj_close_file(file);
3763 3783 return ((struct _buf *)-1);
3764 3784 }
3765 3785 } else {
3766 3786 file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3767 3787 file->_bsize = cbuf.blksize;
3768 3788 }
3769 3789 }
3770 3790 return (file);
3771 3791 }
3772 3792
3773 3793 static int
3774 3794 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3775 3795 {
3776 3796 struct comphdr *hdr;
3777 3797
3778 3798 /*
3779 3799 * read the compressed image into memory,
3780 3800 * so we can deompress from there
3781 3801 */
3782 3802 file->_dsize = cip->fsize;
3783 3803 file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3784 3804 if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3785 3805 kobj_free(file->_dbuf, cip->fsize);
3786 3806 return (-1);
3787 3807 }
3788 3808
3789 3809 hdr = kobj_comphdr(file);
3790 3810 if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3791 3811 hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3792 3812 !ISP2(hdr->ch_blksize)) {
3793 3813 kobj_free(file->_dbuf, cip->fsize);
3794 3814 return (-1);
3795 3815 }
3796 3816 file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3797 3817 file->_bsize = hdr->ch_blksize;
3798 3818 return (0);
3799 3819 }
3800 3820
3801 3821 void
3802 3822 kobj_close_file(struct _buf *file)
3803 3823 {
3804 3824 kobj_close(file->_fd);
3805 3825 if (file->_base != NULL)
3806 3826 kobj_free(file->_base, file->_bsize);
3807 3827 if (file->_dbuf != NULL)
3808 3828 kobj_free(file->_dbuf, file->_dsize);
3809 3829 kobj_free(file->_name, strlen(file->_name)+1);
3810 3830 kobj_free(file, sizeof (struct _buf));
3811 3831 }
3812 3832
3813 3833 int
3814 3834 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3815 3835 {
3816 3836 int b_size, c_size;
3817 3837 int b_off; /* Offset into buffer for start of bcopy */
3818 3838 int count = 0;
3819 3839 int page_addr;
3820 3840
3821 3841 if (_moddebug & MODDEBUG_ERRMSG) {
3822 3842 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3823 3843 _kobj_printf(ops, " offset=%x at", off);
3824 3844 _kobj_printf(ops, " buf=%x\n", buf);
3825 3845 }
3826 3846
3827 3847 /*
3828 3848 * Handle compressed (gzip for now) file here. First get the
3829 3849 * compressed size, then read the image into memory and finally
3830 3850 * call zlib to decompress the image at the supplied memory buffer.
3831 3851 */
3832 3852 if (file->_iscmp == CH_MAGIC_GZIP) {
3833 3853 ulong_t dlen;
3834 3854 vattr_t vattr;
3835 3855 struct vnode *vp = (struct vnode *)file->_fd;
3836 3856 ssize_t resid;
3837 3857 int err = 0;
3838 3858
3839 3859 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3840 3860 return (-1);
3841 3861
3842 3862 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3843 3863 file->_dsize = vattr.va_size;
3844 3864
3845 3865 /* Read the compressed file into memory */
3846 3866 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3847 3867 (offset_t)(0), UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3848 3868 &resid)) != 0) {
3849 3869
3850 3870 _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3851 3871 "error code 0x%x\n", err);
3852 3872 return (-1);
3853 3873 }
3854 3874
3855 3875 dlen = size;
3856 3876
3857 3877 /* Decompress the image at the supplied memory buffer */
3858 3878 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3859 3879 vattr.va_size)) != Z_OK) {
3860 3880 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3861 3881 "failed, error code : 0x%x\n", err);
3862 3882 return (-1);
3863 3883 }
3864 3884
3865 3885 if (dlen != size) {
3866 3886 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3867 3887 "failed to uncompress (size returned 0x%x , "
3868 3888 "expected size: 0x%x)\n", dlen, size);
3869 3889 return (-1);
3870 3890 }
3871 3891
3872 3892 return (0);
3873 3893 }
3874 3894
3875 3895 while (size) {
3876 3896 page_addr = F_PAGE(file, off);
3877 3897 b_size = file->_size;
3878 3898 /*
3879 3899 * If we have the filesystem page the caller's referring to
3880 3900 * and we have something in the buffer,
3881 3901 * satisfy as much of the request from the buffer as we can.
3882 3902 */
3883 3903 if (page_addr == file->_off && b_size > 0) {
3884 3904 b_off = B_OFFSET(file, off);
3885 3905 c_size = b_size - b_off;
3886 3906 /*
3887 3907 * If there's nothing to copy, we're at EOF.
3888 3908 */
3889 3909 if (c_size <= 0)
3890 3910 break;
3891 3911 if (c_size > size)
3892 3912 c_size = size;
3893 3913 if (buf) {
3894 3914 if (_moddebug & MODDEBUG_ERRMSG)
3895 3915 _kobj_printf(ops, "copying %x bytes\n",
3896 3916 c_size);
3897 3917 bcopy(file->_base+b_off, buf, c_size);
3898 3918 size -= c_size;
3899 3919 off += c_size;
3900 3920 buf += c_size;
3901 3921 count += c_size;
3902 3922 } else {
3903 3923 _kobj_printf(ops, "kobj_read: system error");
3904 3924 count = -1;
3905 3925 break;
3906 3926 }
3907 3927 } else {
3908 3928 /*
3909 3929 * If the caller's offset is page aligned and
3910 3930 * the caller want's at least a filesystem page and
3911 3931 * the caller provided a buffer,
3912 3932 * read directly into the caller's buffer.
3913 3933 */
3914 3934 if (page_addr == off &&
3915 3935 (c_size = F_BLKS(file, size)) && buf) {
3916 3936 c_size = kobj_read_blks(file, buf, c_size,
3917 3937 page_addr);
3918 3938 if (c_size < 0) {
3919 3939 count = -1;
3920 3940 break;
3921 3941 }
3922 3942 count += c_size;
3923 3943 if (c_size != F_BLKS(file, size))
3924 3944 break;
3925 3945 size -= c_size;
3926 3946 off += c_size;
3927 3947 buf += c_size;
3928 3948 /*
3929 3949 * Otherwise, read into our buffer and copy next time
3930 3950 * around the loop.
3931 3951 */
3932 3952 } else {
3933 3953 file->_off = page_addr;
3934 3954 c_size = kobj_read_blks(file, file->_base,
3935 3955 file->_bsize, page_addr);
3936 3956 file->_ptr = file->_base;
3937 3957 file->_cnt = c_size;
3938 3958 file->_size = c_size;
3939 3959 /*
3940 3960 * If a _filbuf call or nothing read, break.
3941 3961 */
3942 3962 if (buf == NULL || c_size <= 0) {
3943 3963 count = c_size;
3944 3964 break;
3945 3965 }
3946 3966 }
3947 3967 if (_moddebug & MODDEBUG_ERRMSG)
3948 3968 _kobj_printf(ops, "read %x bytes\n", c_size);
3949 3969 }
3950 3970 }
3951 3971 if (_moddebug & MODDEBUG_ERRMSG)
3952 3972 _kobj_printf(ops, "count = %x\n", count);
3953 3973
3954 3974 return (count);
3955 3975 }
3956 3976
3957 3977 static int
3958 3978 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3959 3979 {
3960 3980 int ret;
3961 3981
3962 3982 ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3963 3983 if (file->_iscmp) {
3964 3984 uint_t blks;
3965 3985 int nret;
3966 3986
3967 3987 ret = 0;
3968 3988 for (blks = size / file->_bsize; blks != 0; blks--) {
3969 3989 nret = kobj_uncomp_blk(file, buf, off);
3970 3990 if (nret == -1)
3971 3991 return (-1);
3972 3992 buf += nret;
3973 3993 off += nret;
3974 3994 ret += nret;
3975 3995 if (nret < file->_bsize)
3976 3996 break;
3977 3997 }
3978 3998 } else
3979 3999 ret = kobj_read(file->_fd, buf, size, off);
3980 4000 return (ret);
3981 4001 }
3982 4002
3983 4003 static int
3984 4004 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
3985 4005 {
3986 4006 struct comphdr *hdr = kobj_comphdr(file);
3987 4007 ulong_t dlen, slen;
3988 4008 caddr_t src;
3989 4009 int i;
3990 4010
3991 4011 dlen = file->_bsize;
3992 4012 i = off / file->_bsize;
3993 4013 src = file->_dbuf + hdr->ch_blkmap[i];
3994 4014 if (i == hdr->ch_fsize / file->_bsize)
3995 4015 slen = file->_dsize - hdr->ch_blkmap[i];
3996 4016 else
3997 4017 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
3998 4018 if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
3999 4019 return (-1);
4000 4020 return (dlen);
4001 4021 }
4002 4022
4003 4023 int
4004 4024 kobj_filbuf(struct _buf *f)
4005 4025 {
4006 4026 if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
4007 4027 return (kobj_getc(f));
4008 4028 return (-1);
4009 4029 }
4010 4030
4011 4031 void
4012 4032 kobj_free(void *address, size_t size)
4013 4033 {
4014 4034 if (standalone)
4015 4035 return;
4016 4036
4017 4037 kmem_free(address, size);
4018 4038 kobj_stat.nfree_calls++;
4019 4039 kobj_stat.nfree += size;
4020 4040 }
4021 4041
4022 4042 void *
4023 4043 kobj_zalloc(size_t size, int flag)
4024 4044 {
4025 4045 void *v;
4026 4046
4027 4047 if ((v = kobj_alloc(size, flag)) != 0) {
4028 4048 bzero(v, size);
4029 4049 }
4030 4050
4031 4051 return (v);
4032 4052 }
4033 4053
4034 4054 void *
4035 4055 kobj_alloc(size_t size, int flag)
4036 4056 {
4037 4057 /*
4038 4058 * If we are running standalone in the
4039 4059 * linker, we ask boot for memory.
4040 4060 * Either it's temporary memory that we lose
4041 4061 * once boot is mapped out or we allocate it
4042 4062 * permanently using the dynamic data segment.
4043 4063 */
4044 4064 if (standalone) {
4045 4065 #if defined(_OBP)
4046 4066 if (flag & (KM_TMP | KM_SCRATCH))
4047 4067 return (bop_temp_alloc(size, MINALIGN));
4048 4068 #else
4049 4069 if (flag & (KM_TMP | KM_SCRATCH))
4050 4070 return (BOP_ALLOC(ops, 0, size, MINALIGN));
4051 4071 #endif
4052 4072 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
4053 4073 }
4054 4074
4055 4075 kobj_stat.nalloc_calls++;
4056 4076 kobj_stat.nalloc += size;
4057 4077
4058 4078 return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
4059 4079 }
4060 4080
4061 4081 /*
4062 4082 * Allow the "mod" system to sync up with the work
4063 4083 * already done by kobj during the initial loading
4064 4084 * of the kernel. This also gives us a chance
4065 4085 * to reallocate memory that belongs to boot.
4066 4086 */
4067 4087 void
4068 4088 kobj_sync(void)
4069 4089 {
4070 4090 struct modctl_list *lp, **lpp;
4071 4091
4072 4092 /*
4073 4093 * The module path can be set in /etc/system via 'moddir' commands
4074 4094 */
4075 4095 if (default_path != NULL)
4076 4096 kobj_module_path = default_path;
4077 4097 else
4078 4098 default_path = kobj_module_path;
4079 4099
4080 4100 ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
4081 4101 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4082 4102
4083 4103 ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
4084 4104 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4085 4105
4086 4106 /*
4087 4107 * Move symbol tables from boot memory to ksyms_arena.
4088 4108 */
4089 4109 for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
4090 4110 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
4091 4111 kobj_export_module(mod(lp));
4092 4112 }
4093 4113 }
4094 4114
4095 4115 caddr_t
4096 4116 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
4097 4117 {
4098 4118 uintptr_t va, pva;
4099 4119 size_t alloc_pgsz = kobj_mmu_pagesize;
4100 4120 size_t alloc_align = BO_NO_ALIGN;
4101 4121 size_t alloc_size;
4102 4122
4103 4123 /*
4104 4124 * If we are using "large" mappings for the kernel,
4105 4125 * request aligned memory from boot using the
4106 4126 * "large" pagesize.
4107 4127 */
4108 4128 if (lg_pagesize) {
4109 4129 alloc_align = lg_pagesize;
4110 4130 alloc_pgsz = lg_pagesize;
4111 4131 }
4112 4132
4113 4133 #if defined(__sparc)
4114 4134 /* account for redzone */
4115 4135 if (limit)
4116 4136 limit -= alloc_pgsz;
4117 4137 #endif /* __sparc */
4118 4138
4119 4139 va = ALIGN((uintptr_t)*spp, align);
4120 4140 pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4121 4141 /*
4122 4142 * Need more pages?
4123 4143 */
4124 4144 if (va + size > pva) {
4125 4145 uintptr_t npva;
4126 4146
4127 4147 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4128 4148 /*
4129 4149 * Check for overlapping segments.
4130 4150 */
4131 4151 if (limit && limit <= *spp + alloc_size) {
4132 4152 return ((caddr_t)0);
4133 4153 }
4134 4154
4135 4155 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4136 4156 alloc_size, alloc_align);
4137 4157
4138 4158 if (npva == 0) {
4139 4159 _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4140 4160 alloc_size);
4141 4161 _kobj_printf(ops, " aligned %lx", alloc_align);
4142 4162 _kobj_printf(ops, " at 0x%lx\n", pva);
4143 4163 return (NULL);
4144 4164 }
4145 4165 }
4146 4166 *spp = (caddr_t)(va + size);
4147 4167
4148 4168 return ((caddr_t)va);
4149 4169 }
4150 4170
4151 4171 /*
4152 4172 * Calculate the number of output hash buckets.
4153 4173 * We use the next prime larger than n / 4,
4154 4174 * so the average hash chain is about 4 entries.
4155 4175 * More buckets would just be a waste of memory.
4156 4176 */
4157 4177 uint_t
4158 4178 kobj_gethashsize(uint_t n)
4159 4179 {
4160 4180 int f;
4161 4181 int hsize = MAX(n / 4, 2);
4162 4182
4163 4183 for (f = 2; f * f <= hsize; f++)
4164 4184 if (hsize % f == 0)
4165 4185 hsize += f = 1;
4166 4186
4167 4187 return (hsize);
4168 4188 }
4169 4189
4170 4190 /*
4171 4191 * Get the file size.
4172 4192 *
4173 4193 * Before root is mounted, files are compressed in the boot_archive ramdisk
4174 4194 * (in the memory). kobj_fstat would return the compressed file size.
4175 4195 * In order to get the uncompressed file size, read the file to the end and
4176 4196 * count its size.
4177 4197 */
4178 4198 int
4179 4199 kobj_get_filesize(struct _buf *file, uint64_t *size)
4180 4200 {
4181 4201 int err = 0;
4182 4202 ssize_t resid;
4183 4203 uint32_t buf;
4184 4204
4185 4205 if (_modrootloaded) {
4186 4206 struct bootstat bst;
4187 4207
4188 4208 if (kobj_fstat(file->_fd, &bst) != 0)
4189 4209 return (EIO);
4190 4210 *size = bst.st_size;
4191 4211
4192 4212 if (file->_iscmp == CH_MAGIC_GZIP) {
4193 4213 /*
4194 4214 * Read the last 4 bytes of the compressed (gzip)
4195 4215 * image to get the size of its uncompressed
4196 4216 * version.
4197 4217 */
4198 4218 if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4199 4219 (char *)(&buf), 4, (offset_t)(*size - 4),
4200 4220 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid))
4201 4221 != 0) {
4202 4222 _kobj_printf(ops, "kobj_get_filesize: "
4203 4223 "vn_rdwr() failed with error 0x%x\n", err);
4204 4224 return (-1);
4205 4225 }
4206 4226
4207 4227 *size = (uint64_t)buf;
4208 4228 }
4209 4229 } else {
4210 4230
4211 4231 #if defined(_OBP)
4212 4232 struct bootstat bsb;
4213 4233
4214 4234 if (file->_iscmp) {
4215 4235 struct comphdr *hdr = kobj_comphdr(file);
4216 4236
4217 4237 *size = hdr->ch_fsize;
4218 4238 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4219 4239 return (EIO);
4220 4240 else
4221 4241 *size = bsb.st_size;
4222 4242 #else
4223 4243 char *buf;
4224 4244 int count;
4225 4245 uint64_t offset = 0;
4226 4246
4227 4247 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4228 4248 do {
4229 4249 count = kobj_read_file(file, buf, MAXBSIZE, offset);
4230 4250 if (count < 0) {
4231 4251 kmem_free(buf, MAXBSIZE);
4232 4252 return (EIO);
4233 4253 }
4234 4254 offset += count;
4235 4255 } while (count == MAXBSIZE);
4236 4256 kmem_free(buf, MAXBSIZE);
4237 4257
4238 4258 *size = offset;
4239 4259 #endif
4240 4260 }
4241 4261
4242 4262 return (0);
4243 4263 }
4244 4264
4245 4265 static char *
4246 4266 basename(char *s)
4247 4267 {
4248 4268 char *p, *q;
4249 4269
4250 4270 q = NULL;
4251 4271 p = s;
4252 4272 do {
4253 4273 if (*p == '/')
4254 4274 q = p;
4255 4275 } while (*p++);
4256 4276 return (q ? q + 1 : s);
4257 4277 }
4258 4278
4259 4279 void
4260 4280 kobj_stat_get(kobj_stat_t *kp)
4261 4281 {
4262 4282 *kp = kobj_stat;
4263 4283 }
4264 4284
4265 4285 int
4266 4286 kobj_getpagesize()
4267 4287 {
4268 4288 return (lg_pagesize);
4269 4289 }
4270 4290
4271 4291 void
4272 4292 kobj_textwin_alloc(struct module *mp)
4273 4293 {
4274 4294 ASSERT(MUTEX_HELD(&mod_lock));
4275 4295
4276 4296 if (mp->textwin != NULL)
4277 4297 return;
4278 4298
4279 4299 /*
4280 4300 * If the text is not contained in the heap, then it is not contained
4281 4301 * by a writable mapping. (Specifically, it's on the nucleus page.)
4282 4302 * We allocate a read/write mapping for this module's text to allow
4283 4303 * the text to be patched without calling hot_patch_kernel_text()
4284 4304 * (which is quite slow).
4285 4305 */
4286 4306 if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4287 4307 uintptr_t text = (uintptr_t)mp->text;
4288 4308 uintptr_t size = (uintptr_t)mp->text_size;
4289 4309 uintptr_t i;
4290 4310 caddr_t va;
4291 4311 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4292 4312 (text & PAGEMASK);
4293 4313
4294 4314 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4295 4315
4296 4316 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4297 4317 hat_devload(kas.a_hat, va, PAGESIZE,
4298 4318 hat_getpfnum(kas.a_hat, (caddr_t)i),
4299 4319 PROT_READ | PROT_WRITE,
4300 4320 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4301 4321 va += PAGESIZE;
4302 4322 }
4303 4323
4304 4324 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4305 4325 } else {
4306 4326 mp->textwin = mp->text;
4307 4327 }
4308 4328 }
4309 4329
4310 4330 void
4311 4331 kobj_textwin_free(struct module *mp)
4312 4332 {
4313 4333 uintptr_t text = (uintptr_t)mp->text;
4314 4334 uintptr_t tsize = (uintptr_t)mp->text_size;
4315 4335 size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4316 4336 (text & PAGEMASK));
4317 4337
4318 4338 mp->textwin = NULL;
4319 4339
4320 4340 if (mp->textwin_base == NULL)
4321 4341 return;
4322 4342
4323 4343 hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4324 4344 vmem_free(heap_arena, mp->textwin_base, size);
4325 4345 mp->textwin_base = NULL;
4326 4346 }
4327 4347
4328 4348 static char *
4329 4349 find_libmacro(char *name)
4330 4350 {
4331 4351 int lmi;
4332 4352
4333 4353 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4334 4354 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4335 4355 return (libmacros[lmi].lmi_list);
4336 4356 }
4337 4357 return (NULL);
4338 4358 }
4339 4359
4340 4360 /*
4341 4361 * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4342 4362 * returns path if successful, else NULL
4343 4363 * Support multiple $MACROs expansion and the first valid path will be returned
4344 4364 * Caller's responsibility to provide enough space in path to expand
4345 4365 */
4346 4366 char *
4347 4367 expand_libmacro(char *tail, char *path, char *pathend)
4348 4368 {
4349 4369 char c, *p, *p1, *p2, *path2, *endp;
4350 4370 int diff, lmi, macrolen, valid_macro, more_macro;
4351 4371 struct _buf *file;
4352 4372
4353 4373 /*
4354 4374 * check for $MACROS between nulls or slashes
4355 4375 */
4356 4376 p = strchr(tail, '$');
4357 4377 if (p == NULL)
4358 4378 return (NULL);
4359 4379 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4360 4380 macrolen = libmacros[lmi].lmi_macrolen;
4361 4381 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4362 4382 break;
4363 4383 }
4364 4384
4365 4385 valid_macro = 0;
4366 4386 if (lmi < NLIBMACROS) {
4367 4387 /*
4368 4388 * The following checks are used to restrict expansion of
4369 4389 * macros to those that form a full directory/file name
4370 4390 * and to keep the behavior same as before. If this
4371 4391 * restriction is removed or no longer valid in the future,
4372 4392 * the checks below can be deleted.
4373 4393 */
4374 4394 if ((p == tail) || (*(p - 1) == '/')) {
4375 4395 c = *(p + macrolen + 1);
4376 4396 if (c == '/' || c == '\0')
4377 4397 valid_macro = 1;
4378 4398 }
4379 4399 }
4380 4400
4381 4401 if (!valid_macro) {
4382 4402 p2 = strchr(p, '/');
4383 4403 /*
4384 4404 * if no more macro to expand, then just copy whatever left
4385 4405 * and check whether it exists
4386 4406 */
4387 4407 if (p2 == NULL || strchr(p2, '$') == NULL) {
4388 4408 (void) strcpy(pathend, tail);
4389 4409 if ((file = kobj_open_path(path, 1, 1)) !=
4390 4410 (struct _buf *)-1) {
4391 4411 kobj_close_file(file);
4392 4412 return (path);
4393 4413 } else
4394 4414 return (NULL);
4395 4415 } else {
4396 4416 /*
4397 4417 * copy all chars before '/' and call expand_libmacro()
4398 4418 * again
4399 4419 */
4400 4420 diff = p2 - tail;
4401 4421 bcopy(tail, pathend, diff);
4402 4422 pathend += diff;
4403 4423 *(pathend) = '\0';
4404 4424 return (expand_libmacro(p2, path, pathend));
4405 4425 }
4406 4426 }
4407 4427
4408 4428 more_macro = 0;
4409 4429 if (c != '\0') {
4410 4430 endp = p + macrolen + 1;
4411 4431 if (strchr(endp, '$') != NULL)
4412 4432 more_macro = 1;
4413 4433 } else
4414 4434 endp = NULL;
4415 4435
4416 4436 /*
4417 4437 * copy lmi_list and split it into components.
4418 4438 * then put the part of tail before $MACRO into path
4419 4439 * at pathend
4420 4440 */
4421 4441 diff = p - tail;
4422 4442 if (diff > 0)
4423 4443 bcopy(tail, pathend, diff);
4424 4444 path2 = pathend + diff;
4425 4445 p1 = libmacros[lmi].lmi_list;
4426 4446 while (p1 && (*p1 != '\0')) {
4427 4447 p2 = strchr(p1, ':');
4428 4448 if (p2) {
4429 4449 diff = p2 - p1;
4430 4450 bcopy(p1, path2, diff);
4431 4451 *(path2 + diff) = '\0';
4432 4452 } else {
4433 4453 diff = strlen(p1);
4434 4454 bcopy(p1, path2, diff + 1);
4435 4455 }
4436 4456 /* copy endp only if there isn't any more macro to expand */
4437 4457 if (!more_macro && (endp != NULL))
4438 4458 (void) strcat(path2, endp);
4439 4459 file = kobj_open_path(path, 1, 1);
4440 4460 if (file != (struct _buf *)-1) {
4441 4461 kobj_close_file(file);
4442 4462 /*
4443 4463 * if more macros to expand then call expand_libmacro(),
4444 4464 * else return path which has the whole path
4445 4465 */
4446 4466 if (!more_macro || (expand_libmacro(endp, path,
4447 4467 path2 + diff) != NULL)) {
4448 4468 return (path);
4449 4469 }
4450 4470 }
4451 4471 if (p2)
4452 4472 p1 = ++p2;
4453 4473 else
4454 4474 return (NULL);
4455 4475 }
4456 4476 return (NULL);
4457 4477 }
4458 4478
4459 4479 static void
4460 4480 tnf_add_notifyunload(kobj_notify_f *fp)
4461 4481 {
4462 4482 kobj_notify_list_t *entry;
4463 4483
4464 4484 entry = kobj_alloc(sizeof (kobj_notify_list_t), KM_WAIT);
4465 4485 entry->kn_type = KOBJ_NOTIFY_MODUNLOADING;
4466 4486 entry->kn_func = fp;
4467 4487 (void) kobj_notify_add(entry);
4468 4488 }
4469 4489
4470 4490 /* ARGSUSED */
4471 4491 static void
4472 4492 tnf_unsplice_probes(uint_t what, struct modctl *mod)
4473 4493 {
4474 4494 tnf_probe_control_t **p;
4475 4495 tnf_tag_data_t **q;
4476 4496 struct module *mp = mod->mod_mp;
4477 4497
4478 4498 if (!(mp->flags & KOBJ_TNF_PROBE))
4479 4499 return;
4480 4500
4481 4501 for (p = &__tnf_probe_list_head; *p; )
4482 4502 if (kobj_addrcheck(mp, (char *)*p) == 0)
4483 4503 *p = (*p)->next;
4484 4504 else
4485 4505 p = &(*p)->next;
4486 4506
4487 4507 for (q = &__tnf_tag_list_head; *q; )
4488 4508 if (kobj_addrcheck(mp, (char *)*q) == 0)
4489 4509 *q = (tnf_tag_data_t *)(*q)->tag_version;
4490 4510 else
4491 4511 q = (tnf_tag_data_t **)&(*q)->tag_version;
4492 4512
4493 4513 tnf_changed_probe_list = 1;
4494 4514 }
4495 4515
4496 4516 int
4497 4517 tnf_splice_probes(int boot_load, tnf_probe_control_t *plist,
4498 4518 tnf_tag_data_t *tlist)
4499 4519 {
4500 4520 int result = 0;
4501 4521 static int add_notify = 1;
4502 4522
4503 4523 if (plist) {
4504 4524 tnf_probe_control_t *pl;
4505 4525
4506 4526 for (pl = plist; pl->next; )
4507 4527 pl = pl->next;
4508 4528
4509 4529 if (!boot_load)
4510 4530 mutex_enter(&mod_lock);
4511 4531 tnf_changed_probe_list = 1;
4512 4532 pl->next = __tnf_probe_list_head;
4513 4533 __tnf_probe_list_head = plist;
4514 4534 if (!boot_load)
4515 4535 mutex_exit(&mod_lock);
4516 4536 result = 1;
4517 4537 }
4518 4538
4519 4539 if (tlist) {
4520 4540 tnf_tag_data_t *tl;
4521 4541
4522 4542 for (tl = tlist; tl->tag_version; )
4523 4543 tl = (tnf_tag_data_t *)tl->tag_version;
4524 4544
4525 4545 if (!boot_load)
4526 4546 mutex_enter(&mod_lock);
4527 4547 tl->tag_version = (tnf_tag_version_t *)__tnf_tag_list_head;
4528 4548 __tnf_tag_list_head = tlist;
4529 4549 if (!boot_load)
4530 4550 mutex_exit(&mod_lock);
4531 4551 result = 1;
4532 4552 }
4533 4553 if (!boot_load && result && add_notify) {
4534 4554 tnf_add_notifyunload(tnf_unsplice_probes);
4535 4555 add_notify = 0;
4536 4556 }
4537 4557 return (result);
4538 4558 }
4539 4559
4540 4560 char *kobj_file_buf;
4541 4561 int kobj_file_bufsize;
4542 4562
4543 4563 /*
4544 4564 * This code is for the purpose of manually recording which files
4545 4565 * needs to go into the boot archive on any given system.
4546 4566 *
4547 4567 * To enable the code, set kobj_file_bufsize in /etc/system
4548 4568 * and reboot the system, then use mdb to look at kobj_file_buf.
4549 4569 */
4550 4570 static void
4551 4571 kobj_record_file(char *filename)
4552 4572 {
4553 4573 static char *buf;
4554 4574 static int size = 0;
4555 4575 int n;
4556 4576
4557 4577 if (kobj_file_bufsize == 0) /* don't bother */
4558 4578 return;
4559 4579
4560 4580 if (kobj_file_buf == NULL) { /* allocate buffer */
4561 4581 size = kobj_file_bufsize;
4562 4582 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4563 4583 }
4564 4584
4565 4585 n = snprintf(buf, size, "%s\n", filename);
4566 4586 if (n > size)
4567 4587 n = size;
4568 4588 size -= n;
4569 4589 buf += n;
4570 4590 }
4571 4591
4572 4592 static int
4573 4593 kobj_boot_fstat(int fd, struct bootstat *stp)
4574 4594 {
4575 4595 #if defined(_OBP)
4576 4596 if (!standalone && _ioquiesced)
4577 4597 return (-1);
4578 4598 return (BOP_FSTAT(ops, fd, stp));
4579 4599 #else
4580 4600 return (BRD_FSTAT(bfs_ops, fd, stp));
4581 4601 #endif
4582 4602 }
4583 4603
4584 4604 static int
4585 4605 kobj_boot_open(char *filename, int flags)
4586 4606 {
4587 4607 #if defined(_OBP)
4588 4608
4589 4609 /*
4590 4610 * If io via bootops is quiesced, it means boot is no longer
4591 4611 * available to us. We make it look as if we can't open the
4592 4612 * named file - which is reasonably accurate.
4593 4613 */
4594 4614 if (!standalone && _ioquiesced)
4595 4615 return (-1);
4596 4616
4597 4617 kobj_record_file(filename);
4598 4618 return (BOP_OPEN(filename, flags));
4599 4619 #else /* x86 */
4600 4620 kobj_record_file(filename);
4601 4621 return (BRD_OPEN(bfs_ops, filename, flags));
4602 4622 #endif
4603 4623 }
4604 4624
4605 4625 static int
4606 4626 kobj_boot_close(int fd)
4607 4627 {
4608 4628 #if defined(_OBP)
4609 4629 if (!standalone && _ioquiesced)
4610 4630 return (-1);
4611 4631
4612 4632 return (BOP_CLOSE(fd));
4613 4633 #else /* x86 */
4614 4634 return (BRD_CLOSE(bfs_ops, fd));
4615 4635 #endif
4616 4636 }
4617 4637
4618 4638 /*ARGSUSED*/
4619 4639 static int
4620 4640 kobj_boot_seek(int fd, off_t hi, off_t lo)
4621 4641 {
4622 4642 #if defined(_OBP)
4623 4643 return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4624 4644 #else
4625 4645 return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4626 4646 #endif
4627 4647 }
4628 4648
4629 4649 static int
4630 4650 kobj_boot_read(int fd, caddr_t buf, size_t size)
4631 4651 {
4632 4652 #if defined(_OBP)
4633 4653 return (BOP_READ(fd, buf, size));
4634 4654 #else
4635 4655 return (BRD_READ(bfs_ops, fd, buf, size));
4636 4656 #endif
4637 4657 }
4638 4658
4639 4659 static int
4640 4660 kobj_boot_compinfo(int fd, struct compinfo *cb)
4641 4661 {
4642 4662 return (boot_compinfo(fd, cb));
4643 4663 }
4644 4664
4645 4665 /*
4646 4666 * Check if the file is compressed (for now we handle only gzip).
4647 4667 * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4648 4668 */
4649 4669 static int
4650 4670 kobj_is_compressed(intptr_t fd)
4651 4671 {
4652 4672 struct vnode *vp = (struct vnode *)fd;
4653 4673 ssize_t resid;
4654 4674 uint16_t magic_buf;
4655 4675 int err = 0;
4656 4676
4657 4677 if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4658 4678 sizeof (magic_buf), (offset_t)(0),
4659 4679 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
4660 4680
4661 4681 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4662 4682 "error code 0x%x\n", err);
4663 4683 return (0);
4664 4684 }
4665 4685
4666 4686 if (magic_buf == CH_MAGIC_GZIP)
4667 4687 return (CH_MAGIC_GZIP);
4668 4688
4669 4689 return (0);
4670 4690 }
|
↓ open down ↓ |
1621 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX