1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
27 * All rights reserved. Use is subject to license terms.
28 * Copyright 2020 Joyent, Inc.
29 */
30
31 /*
32 * Kernel's linker/loader
33 */
34
35 #include <sys/types.h>
36 #include <sys/param.h>
37 #include <sys/sysmacros.h>
38 #include <sys/systm.h>
39 #include <sys/user.h>
40 #include <sys/kmem.h>
41 #include <sys/reboot.h>
42 #include <sys/bootconf.h>
43 #include <sys/debug.h>
44 #include <sys/uio.h>
45 #include <sys/file.h>
46 #include <sys/vnode.h>
47 #include <sys/user.h>
48 #include <sys/mman.h>
49 #include <vm/as.h>
50 #include <vm/seg_kp.h>
51 #include <vm/seg_kmem.h>
52 #include <sys/elf.h>
53 #include <sys/elf_notes.h>
54 #include <sys/vmsystm.h>
55 #include <sys/kdi.h>
56 #include <sys/atomic.h>
57 #include <sys/kmdb.h>
58
59 #include <sys/link.h>
60 #include <sys/kobj.h>
61 #include <sys/ksyms.h>
62 #include <sys/disp.h>
63 #include <sys/modctl.h>
64 #include <sys/varargs.h>
65 #include <sys/kstat.h>
66 #include <sys/kobj_impl.h>
67 #include <sys/fs/decomp.h>
68 #include <sys/callb.h>
69 #include <sys/cmn_err.h>
70 #include <sys/tnf_probe.h>
71 #include <sys/zmod.h>
72
73 #include <krtld/reloc.h>
74 #include <krtld/kobj_kdi.h>
75 #include <sys/sha1.h>
76 #include <sys/crypto/elfsign.h>
77
78 #if !defined(_OBP)
79 #include <sys/bootvfs.h>
80 #endif
81
82 /*
83 * do_symbols() error codes
84 */
85 #define DOSYM_UNDEF -1 /* undefined symbol */
86 #define DOSYM_UNSAFE -2 /* MT-unsafe driver symbol */
87
88 #if !defined(_OBP)
89 static void synthetic_bootaux(char *, val_t *);
90 #endif
91
92 static struct module *load_exec(val_t *, char *);
93 static void load_linker(val_t *);
94 static struct modctl *add_primary(const char *filename, int);
95 static int bind_primary(val_t *, int);
96 static int load_primary(struct module *, int);
97 static int load_kmdb(val_t *);
98 static int get_progbits(struct module *, struct _buf *);
99 static int get_syms(struct module *, struct _buf *);
100 static int get_ctf(struct module *, struct _buf *);
101 static void get_signature(struct module *, struct _buf *);
102 static int do_common(struct module *);
103 static void add_dependent(struct module *, struct module *);
104 static int do_dependents(struct modctl *, char *, size_t);
105 static int do_symbols(struct module *, Elf64_Addr);
106 static void module_assign(struct modctl *, struct module *);
107 static void free_module_data(struct module *);
108 static char *depends_on(struct module *);
109 static char *getmodpath(const char *);
110 static char *basename(char *);
111 static void attr_val(val_t *);
112 static char *find_libmacro(char *);
113 static char *expand_libmacro(char *, char *, char *);
114 static int read_bootflags(void);
115 static int kobj_comp_setup(struct _buf *, struct compinfo *);
116 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
117 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
118 static int kobj_boot_open(char *, int);
119 static int kobj_boot_close(int);
120 static int kobj_boot_seek(int, off_t, off_t);
121 static int kobj_boot_read(int, caddr_t, size_t);
122 static int kobj_boot_fstat(int, struct bootstat *);
123 static int kobj_boot_compinfo(int, struct compinfo *);
124
125 static Sym *lookup_one(struct module *, const char *);
126 static void sym_insert(struct module *, char *, symid_t);
127 static Sym *sym_lookup(struct module *, Sym *);
128
129 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
130 static void kobjopen_free(struct kobjopen_tctl *ltp);
131 static void kobjopen_thread(struct kobjopen_tctl *ltp);
132 static int kobj_is_compressed(intptr_t);
133
134 extern int kcopy(const void *, void *, size_t);
135 extern int elf_mach_ok(Ehdr *);
136 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
137
138 #if !defined(_OBP)
139 extern int kobj_boot_mountroot(void);
140 #endif
141
142 static void tnf_unsplice_probes(uint_t, struct modctl *);
143 extern tnf_probe_control_t *__tnf_probe_list_head;
144 extern tnf_tag_data_t *__tnf_tag_list_head;
145
146 extern int modrootloaded;
147 extern int swaploaded;
148 extern int bop_io_quiesced;
149 extern int last_module_id;
150
151 extern char stubs_base[];
152 extern char stubs_end[];
153
154 #ifdef KOBJ_DEBUG
155 /*
156 * Values that can be or'd in to kobj_debug and their effects:
157 *
158 * D_DEBUG - misc. debugging information.
159 * D_SYMBOLS - list symbols and their values as they are entered
160 * into the hash table
161 * D_RELOCATIONS - display relocation processing information
162 * D_LOADING - display information about each module as it
163 * is loaded.
164 */
165 int kobj_debug = 0;
166
167 #define KOBJ_MARK(s) if (kobj_debug & D_DEBUG) \
168 (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
169 #else
170 #define KOBJ_MARK(s) /* discard */
171 #endif
172
173 #define MODPATH_PROPNAME "module-path"
174
175 #ifdef MODDIR_SUFFIX
176 static char slash_moddir_suffix_slash[] = MODDIR_SUFFIX "/";
177 #else
178 #define slash_moddir_suffix_slash ""
179 #endif
180
181 #define _moddebug get_weakish_int(&moddebug)
182 #define _modrootloaded get_weakish_int(&modrootloaded)
183 #define _swaploaded get_weakish_int(&swaploaded)
184 #define _ioquiesced get_weakish_int(&bop_io_quiesced)
185
186 #define mod(X) (struct module *)((X)->modl_modp->mod_mp)
187
188 void *romp; /* rom vector (opaque to us) */
189 struct bootops *ops; /* bootops vector */
190 void *dbvec; /* debug vector */
191
192 /*
193 * kobjopen thread control structure
194 */
195 struct kobjopen_tctl {
196 ksema_t sema;
197 char *name; /* name of file */
198 struct vnode *vp; /* vnode return from vn_open() */
199 int Errno; /* error return from vnopen */
200 };
201
202 /*
203 * Structure for defining dynamically expandable library macros
204 */
205
206 struct lib_macro_info {
207 char *lmi_list; /* ptr to list of possible choices */
208 char *lmi_macroname; /* pointer to macro name */
209 ushort_t lmi_ba_index; /* index into bootaux vector */
210 ushort_t lmi_macrolen; /* macro length */
211 } libmacros[] = {
212 { NULL, "CPU", BA_CPU, 0 },
213 { NULL, "MMU", BA_MMU, 0 }
214 };
215
216 #define NLIBMACROS sizeof (libmacros) / sizeof (struct lib_macro_info)
217
218 char *boot_cpu_compatible_list; /* make $CPU available */
219
220 char *kobj_module_path; /* module search path */
221 vmem_t *text_arena; /* module text arena */
222 static vmem_t *data_arena; /* module data & bss arena */
223 static vmem_t *ctf_arena; /* CTF debug data arena */
224 static struct modctl *kobj_modules = NULL; /* modules loaded */
225 int kobj_mmu_pagesize; /* system pagesize */
226 static int lg_pagesize; /* "large" pagesize */
227 static int kobj_last_module_id = 0; /* id assignment */
228 static kmutex_t kobj_lock; /* protects mach memory list */
229
230 /*
231 * The following functions have been implemented by the kernel.
232 * However, many 3rd party drivers provide their own implementations
233 * of these functions. When such drivers are loaded, messages
234 * indicating that these symbols have been multiply defined will be
235 * emitted to the console. To avoid alarming customers for no good
236 * reason, we simply suppress such warnings for the following set of
237 * functions.
238 */
239 static char *suppress_sym_list[] =
240 {
241 "strstr",
242 "strncat",
243 "strlcat",
244 "strlcpy",
245 "strspn",
246 "memcpy",
247 "memset",
248 "memmove",
249 "memcmp",
250 "memchr",
251 "__udivdi3",
252 "__divdi3",
253 "__umoddi3",
254 "__moddi3",
255 NULL /* This entry must exist */
256 };
257
258 /* indexed by KOBJ_NOTIFY_* */
259 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
260
261 /*
262 * TNF probe management globals
263 */
264 tnf_probe_control_t *__tnf_probe_list_head = NULL;
265 tnf_tag_data_t *__tnf_tag_list_head = NULL;
266 int tnf_changed_probe_list = 0;
267
268 /*
269 * Prefix for statically defined tracing (SDT) DTrace probes.
270 */
271 const char *sdt_prefix = "__dtrace_probe_";
272
273 /*
274 * Beginning and end of the kernel's dynamic text/data segments.
275 */
276 static caddr_t _text;
277 static caddr_t _etext;
278 static caddr_t _data;
279
280 /*
281 * The sparc linker doesn't create a memory location
282 * for a variable named _edata, so _edata can only be
283 * referred to, not modified. krtld needs a static
284 * variable to modify it - within krtld, of course -
285 * outside of krtld, e_data is used in all kernels.
286 */
287 #if defined(__sparc)
288 static caddr_t _edata;
289 #else
290 extern caddr_t _edata;
291 #endif
292
293 Addr dynseg = 0; /* load address of "dynamic" segment */
294 size_t dynsize; /* "dynamic" segment size */
295
296
297 int standalone = 1; /* an unwholey kernel? */
298 int use_iflush; /* iflush after relocations */
299
300 /*
301 * _kobj_printf() and _vkobj_printf()
302 *
303 * Common printf function pointer. Can handle only one conversion
304 * specification in the format string. Some of the functions invoked
305 * through this function pointer cannot handle more that one conversion
306 * specification in the format string.
307 */
308 void (*_kobj_printf)(void *, const char *, ...) __KPRINTFLIKE(2);
309 void (*_vkobj_printf)(void *, const char *, va_list) __KVPRINTFLIKE(2);
310
311 /*
312 * Standalone function pointers for use within krtld.
313 * Many platforms implement optimized platmod versions of
314 * utilities such as bcopy and any such are not yet available
315 * until the kernel is more completely stitched together.
316 * See kobj_impl.h
317 */
318 void (*kobj_bcopy)(const void *, void *, size_t);
319 void (*kobj_bzero)(void *, size_t);
320 size_t (*kobj_strlcat)(char *, const char *, size_t);
321
322 static kobj_stat_t kobj_stat;
323
324 #define MINALIGN 8 /* at least a double-word */
325
326 int
327 get_weakish_int(int *ip)
328 {
329 if (standalone)
330 return (0);
331 return (ip == NULL ? 0 : *ip);
332 }
333
334 static void *
335 get_weakish_pointer(void **ptrp)
336 {
337 if (standalone)
338 return (0);
339 return (ptrp == NULL ? 0 : *ptrp);
340 }
341
342 /*
343 * XXX fix dependencies on "kernel"; this should work
344 * for other standalone binaries as well.
345 *
346 * XXX Fix hashing code to use one pointer to
347 * hash entries.
348 * |----------|
349 * | nbuckets |
350 * |----------|
351 * | nchains |
352 * |----------|
353 * | bucket[] |
354 * |----------|
355 * | chain[] |
356 * |----------|
357 */
358
359 /*
360 * Load, bind and relocate all modules that
361 * form the primary kernel. At this point, our
362 * externals have not been relocated.
363 */
364 void
365 kobj_init(
366 void *romvec,
367 void *dvec,
368 struct bootops *bootvec,
369 val_t *bootaux)
370 {
371 struct module *mp;
372 struct modctl *modp;
373 Addr entry;
374 char filename[MAXPATHLEN];
375
376 /*
377 * Save these to pass on to
378 * the booted standalone.
379 */
380 romp = romvec;
381 dbvec = dvec;
382
383 ops = bootvec;
384 kobj_setup_standalone_vectors();
385
386 KOBJ_MARK("Entered kobj_init()");
387
388 (void) BOP_GETPROP(ops, "whoami", filename);
389
390 /*
391 * We don't support standalone debuggers anymore. The use of kadb
392 * will interfere with the later use of kmdb. Let the user mend
393 * their ways now. Users will reach this message if they still
394 * have the kadb binary on their system (perhaps they used an old
395 * bfu, or maybe they intentionally copied it there) and have
396 * specified its use in a way that eluded our checking in the boot
397 * program.
398 */
399 if (dvec != NULL) {
400 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
401 "kadb are no longer supported\n\n");
402 goto fail;
403 }
404
405 #if defined(_OBP)
406 /*
407 * OBP allows us to read both the ramdisk and
408 * the underlying root fs when root is a disk.
409 * This can lower incidences of unbootable systems
410 * when the archive is out-of-date with the /etc
411 * state files.
412 */
413 if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
414 _kobj_printf(ops, "can't mount boot fs\n");
415 goto fail;
416 }
417 #else
418 {
419 /* on x86, we always boot with a ramdisk */
420 (void) kobj_boot_mountroot();
421
422 /*
423 * Now that the ramdisk is mounted, finish boot property
424 * initialization.
425 */
426 read_bootenvrc();
427 }
428
429 #if !defined(_UNIX_KRTLD)
430 /*
431 * 'unix' is linked together with 'krtld' into one executable and
432 * the early boot code does -not- hand us any of the dynamic metadata
433 * about the executable. In particular, it does not read in, map or
434 * otherwise look at the program headers. We fake all that up now.
435 *
436 * We do this early as DTrace static probes and tnf probes both call
437 * undefined references. We have to process those relocations before
438 * calling any of them.
439 *
440 * OBP tells kobj_start() where the ELF image is in memory, so it
441 * synthesized bootaux before kobj_init() was called
442 */
443 if (bootaux[BA_PHDR].ba_ptr == NULL)
444 synthetic_bootaux(filename, bootaux);
445
446 #endif /* !_UNIX_KRTLD */
447 #endif /* _OBP */
448
449 /*
450 * Save the interesting attribute-values
451 * (scanned by kobj_boot).
452 */
453 attr_val(bootaux);
454
455 /*
456 * Set the module search path.
457 */
458 kobj_module_path = getmodpath(filename);
459
460 boot_cpu_compatible_list = find_libmacro("CPU");
461
462 /*
463 * These two modules have actually been
464 * loaded by boot, but we finish the job
465 * by introducing them into the world of
466 * loadable modules.
467 */
468
469 mp = load_exec(bootaux, filename);
470 load_linker(bootaux);
471
472 /*
473 * Load all the primary dependent modules.
474 */
475 if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
476 goto fail;
477
478 /*
479 * Glue it together.
480 */
481 if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
482 goto fail;
483
484 entry = bootaux[BA_ENTRY].ba_val;
485
486 /*
487 * Get the boot flags
488 */
489 bootflags(ops);
490
491 if (boothowto & RB_VERBOSE)
492 kobj_lm_dump(KOBJ_LM_PRIMARY);
493
494 kobj_kdi_init();
495
496 if (boothowto & RB_KMDB) {
497 if (load_kmdb(bootaux) < 0)
498 goto fail;
499 }
500
501 /*
502 * Post setup.
503 */
504 s_text = _text;
505 e_text = _etext;
506 s_data = _data;
507 e_data = _edata;
508
509 kobj_sync_instruction_memory(s_text, e_text - s_text);
510
511 #ifdef KOBJ_DEBUG
512 if (kobj_debug & D_DEBUG)
513 _kobj_printf(ops,
514 "krtld: transferring control to: 0x%lx\n", entry);
515 #endif
516
517 /*
518 * Make sure the mod system knows about the modules already loaded.
519 */
520 last_module_id = kobj_last_module_id;
521 bcopy(kobj_modules, &modules, sizeof (modules));
522 modp = &modules;
523 do {
524 if (modp->mod_next == kobj_modules)
525 modp->mod_next = &modules;
526 if (modp->mod_prev == kobj_modules)
527 modp->mod_prev = &modules;
528 } while ((modp = modp->mod_next) != &modules);
529
530 standalone = 0;
531
532 #ifdef KOBJ_DEBUG
533 if (kobj_debug & D_DEBUG)
534 _kobj_printf(ops,
535 "krtld: really transferring control to: 0x%lx\n", entry);
536 #endif
537
538 /* restore printf/bcopy/bzero vectors before returning */
539 kobj_restore_vectors();
540
541 #if defined(_DBOOT)
542 /*
543 * krtld was called from a dboot ELF section, the embedded
544 * dboot code contains the real entry via bootaux
545 */
546 exitto((caddr_t)entry);
547 #else
548 /*
549 * krtld was directly called from startup
550 */
551 return;
552 #endif
553
554 fail:
555
556 _kobj_printf(ops, "krtld: error during initial load/link phase\n");
557
558 #if !defined(_UNIX_KRTLD)
559 _kobj_printf(ops, "\n");
560 _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
561 " for:\n");
562 _kobj_printf(ops, " %s\n", filename);
563 _kobj_printf(ops, "in the boot archive. Please verify that this"
564 " file\n");
565 _kobj_printf(ops, "matches what is found in the boot archive.\n");
566 _kobj_printf(ops, "You may need to boot using the Solaris failsafe to"
567 " fix this.\n");
568 bop_panic("Unable to boot");
569 #endif
570 }
571
572 #if !defined(_UNIX_KRTLD) && !defined(_OBP)
573 /*
574 * Synthesize additional metadata that describes the executable if
575 * krtld's caller didn't do it.
576 *
577 * (When the dynamic executable has an interpreter, the boot program
578 * does all this for us. Where we don't have an interpreter, (or a
579 * even a boot program, perhaps) we have to do this for ourselves.)
580 */
581 static void
582 synthetic_bootaux(char *filename, val_t *bootaux)
583 {
584 Ehdr ehdr;
585 caddr_t phdrbase;
586 struct _buf *file;
587 int i, n;
588
589 /*
590 * Elf header
591 */
592 KOBJ_MARK("synthetic_bootaux()");
593 KOBJ_MARK(filename);
594 file = kobj_open_file(filename);
595 if (file == (struct _buf *)-1) {
596 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
597 return;
598 }
599 KOBJ_MARK("reading program headers");
600 if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
601 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
602 filename);
603 return;
604 }
605
606 /*
607 * Program headers
608 */
609 bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
610 bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
611 n = ehdr.e_phentsize * ehdr.e_phnum;
612
613 phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
614
615 if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
616 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
617 filename);
618 return;
619 }
620 bootaux[BA_PHDR].ba_ptr = phdrbase;
621 kobj_close_file(file);
622 KOBJ_MARK("closed file");
623
624 /*
625 * Find the dynamic section address
626 */
627 for (i = 0; i < ehdr.e_phnum; i++) {
628 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
629
630 if (phdr->p_type == PT_DYNAMIC) {
631 bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
632 break;
633 }
634 }
635 KOBJ_MARK("synthetic_bootaux() done");
636 }
637 #endif /* !_UNIX_KRTLD && !_OBP */
638
639 /*
640 * Set up any global information derived
641 * from attribute/values in the boot or
642 * aux vector.
643 */
644 static void
645 attr_val(val_t *bootaux)
646 {
647 Phdr *phdr;
648 int phnum, phsize;
649 int i;
650
651 KOBJ_MARK("attr_val()");
652 kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
653 lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
654 use_iflush = bootaux[BA_IFLUSH].ba_val;
655
656 phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
657 phnum = bootaux[BA_PHNUM].ba_val;
658 phsize = bootaux[BA_PHENT].ba_val;
659 for (i = 0; i < phnum; i++) {
660 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
661
662 if (phdr->p_type != PT_LOAD) {
663 continue;
664 }
665 /*
666 * Bounds of the various segments.
667 */
668 if (!(phdr->p_flags & PF_X)) {
669 #if defined(_RELSEG)
670 /*
671 * sparc kernel puts the dynamic info
672 * into a separate segment, which is
673 * free'd in bop_fini()
674 */
675 ASSERT(phdr->p_vaddr != 0);
676 dynseg = phdr->p_vaddr;
677 dynsize = phdr->p_memsz;
678 #else
679 ASSERT(phdr->p_vaddr == 0);
680 #endif
681 } else {
682 if (phdr->p_flags & PF_W) {
683 _data = (caddr_t)phdr->p_vaddr;
684 _edata = _data + phdr->p_memsz;
685 } else {
686 _text = (caddr_t)phdr->p_vaddr;
687 _etext = _text + phdr->p_memsz;
688 }
689 }
690 }
691
692 /* To do the kobj_alloc, _edata needs to be set. */
693 for (i = 0; i < NLIBMACROS; i++) {
694 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
695 libmacros[i].lmi_list = kobj_alloc(
696 strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
697 1, KM_WAIT);
698 (void) strcpy(libmacros[i].lmi_list,
699 bootaux[libmacros[i].lmi_ba_index].ba_ptr);
700 }
701 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
702 }
703 }
704
705 /*
706 * Set up the booted executable.
707 */
708 static struct module *
709 load_exec(val_t *bootaux, char *filename)
710 {
711 struct modctl *cp;
712 struct module *mp;
713 Dyn *dyn;
714 Sym *sp;
715 int i, lsize, osize, nsize, allocsize;
716 char *libname, *tmp;
717 char path[MAXPATHLEN];
718
719 #ifdef KOBJ_DEBUG
720 if (kobj_debug & D_DEBUG)
721 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
722 #endif
723
724 KOBJ_MARK("add_primary");
725 cp = add_primary(filename, KOBJ_LM_PRIMARY);
726
727 KOBJ_MARK("struct module");
728 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
729 cp->mod_mp = mp;
730
731 /*
732 * We don't have the following information
733 * since this module is an executable and not
734 * a relocatable .o.
735 */
736 mp->symtbl_section = 0;
737 mp->shdrs = NULL;
738 mp->strhdr = NULL;
739
740 /*
741 * Since this module is the only exception,
742 * we cons up some section headers.
743 */
744 KOBJ_MARK("symhdr");
745 mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
746
747 KOBJ_MARK("strhdr");
748 mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
749
750 mp->symhdr->sh_type = SHT_SYMTAB;
751 mp->strhdr->sh_type = SHT_STRTAB;
752 /*
753 * Scan the dynamic structure.
754 */
755 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
756 dyn->d_tag != DT_NULL; dyn++) {
757 switch (dyn->d_tag) {
758 case DT_SYMTAB:
759 mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
760 mp->symhdr->sh_addr = dyn->d_un.d_ptr;
761 break;
762 case DT_HASH:
763 mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
764 mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
765 break;
766 case DT_STRTAB:
767 mp->strings = (char *)dyn->d_un.d_ptr;
768 mp->strhdr->sh_addr = dyn->d_un.d_ptr;
769 break;
770 case DT_STRSZ:
771 mp->strhdr->sh_size = dyn->d_un.d_val;
772 break;
773 case DT_SYMENT:
774 mp->symhdr->sh_entsize = dyn->d_un.d_val;
775 break;
776 }
777 }
778
779 /*
780 * Collapse any DT_NEEDED entries into one string.
781 */
782 nsize = osize = 0;
783 allocsize = MAXPATHLEN;
784
785 KOBJ_MARK("depends_on");
786 mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
787
788 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
789 dyn->d_tag != DT_NULL; dyn++)
790 if (dyn->d_tag == DT_NEEDED) {
791 char *_lib;
792
793 libname = mp->strings + dyn->d_un.d_val;
794 if (strchr(libname, '$') != NULL) {
795 if ((_lib = expand_libmacro(libname,
796 path, path)) != NULL)
797 libname = _lib;
798 else
799 _kobj_printf(ops, "krtld: "
800 "load_exec: fail to "
801 "expand %s\n", libname);
802 }
803 lsize = strlen(libname);
804 nsize += lsize;
805 if (nsize + 1 > allocsize) {
806 KOBJ_MARK("grow depends_on");
807 tmp = kobj_alloc(allocsize + MAXPATHLEN,
808 KM_WAIT);
809 bcopy(mp->depends_on, tmp, osize);
810 kobj_free(mp->depends_on, allocsize);
811 mp->depends_on = tmp;
812 allocsize += MAXPATHLEN;
813 }
814 bcopy(libname, mp->depends_on + osize, lsize);
815 *(mp->depends_on + nsize) = ' '; /* separate */
816 nsize++;
817 osize = nsize;
818 }
819 if (nsize) {
820 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
821 /*
822 * alloc with exact size and copy whatever it got over
823 */
824 KOBJ_MARK("realloc depends_on");
825 tmp = kobj_alloc(nsize, KM_WAIT);
826 bcopy(mp->depends_on, tmp, nsize);
827 kobj_free(mp->depends_on, allocsize);
828 mp->depends_on = tmp;
829 } else {
830 kobj_free(mp->depends_on, allocsize);
831 mp->depends_on = NULL;
832 }
833
834 mp->flags = KOBJ_EXEC|KOBJ_PRIM; /* NOT a relocatable .o */
835 mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
836 /*
837 * We allocate our own table since we don't
838 * hash undefined references.
839 */
840 KOBJ_MARK("chains");
841 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
842 KOBJ_MARK("buckets");
843 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
844
845 mp->text = _text;
846 mp->data = _data;
847
848 mp->text_size = _etext - _text;
849 mp->data_size = _edata - _data;
850
851 cp->mod_text = mp->text;
852 cp->mod_text_size = mp->text_size;
853
854 mp->filename = cp->mod_filename;
855
856 #ifdef KOBJ_DEBUG
857 if (kobj_debug & D_LOADING) {
858 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
859 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
860 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
861 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
862 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
863 }
864 #endif /* KOBJ_DEBUG */
865
866 /*
867 * Insert symbols into the hash table.
868 */
869 for (i = 0; i < mp->nsyms; i++) {
870 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
871
872 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
873 continue;
874 #if defined(__sparc)
875 /*
876 * Register symbols are ignored in the kernel
877 */
878 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER)
879 continue;
880 #endif /* __sparc */
881
882 sym_insert(mp, mp->strings + sp->st_name, i);
883 }
884
885 KOBJ_MARK("load_exec done");
886 return (mp);
887 }
888
889 /*
890 * Set up the linker module (if it's compiled in, LDNAME is NULL)
891 */
892 static void
893 load_linker(val_t *bootaux)
894 {
895 struct module *kmp = (struct module *)kobj_modules->mod_mp;
896 struct module *mp;
897 struct modctl *cp;
898 int i;
899 Shdr *shp;
900 Sym *sp;
901 int shsize;
902 char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
903
904 /*
905 * On some architectures, krtld is compiled into the kernel.
906 */
907 if (dlname == NULL)
908 return;
909
910 cp = add_primary(dlname, KOBJ_LM_PRIMARY);
911
912 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
913
914 cp->mod_mp = mp;
915 mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
916 shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
917 mp->shdrs = kobj_alloc(shsize, KM_WAIT);
918 bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
919
920 for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
921 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
922
923 if (shp->sh_flags & SHF_ALLOC) {
924 if (shp->sh_flags & SHF_WRITE) {
925 if (mp->data == NULL)
926 mp->data = (char *)shp->sh_addr;
927 } else if (mp->text == NULL) {
928 mp->text = (char *)shp->sh_addr;
929 }
930 }
931 if (shp->sh_type == SHT_SYMTAB) {
932 mp->symtbl_section = i;
933 mp->symhdr = shp;
934 mp->symspace = mp->symtbl = (char *)shp->sh_addr;
935 }
936 }
937 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
938 mp->flags = KOBJ_INTERP|KOBJ_PRIM;
939 mp->strhdr = (Shdr *)
940 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
941 mp->strings = (char *)mp->strhdr->sh_addr;
942 mp->hashsize = kobj_gethashsize(mp->nsyms);
943
944 mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
945 (mp->hashsize + mp->nsyms) * sizeof (symid_t);
946
947 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
948 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
949
950 mp->bss = bootaux[BA_BSS].ba_val;
951 mp->bss_align = 0; /* pre-aligned during allocation */
952 mp->bss_size = (uintptr_t)_edata - mp->bss;
953 mp->text_size = _etext - mp->text;
954 mp->data_size = _edata - mp->data;
955 mp->filename = cp->mod_filename;
956 cp->mod_text = mp->text;
957 cp->mod_text_size = mp->text_size;
958
959 /*
960 * Now that we've figured out where the linker is,
961 * set the limits for the booted object.
962 */
963 kmp->text_size = (size_t)(mp->text - kmp->text);
964 kmp->data_size = (size_t)(mp->data - kmp->data);
965 kobj_modules->mod_text_size = kmp->text_size;
966
967 #ifdef KOBJ_DEBUG
968 if (kobj_debug & D_LOADING) {
969 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
970 _kobj_printf(ops, "\ttext:0x%p", mp->text);
971 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
972 _kobj_printf(ops, "\tdata:0x%p", mp->data);
973 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
974 }
975 #endif /* KOBJ_DEBUG */
976
977 /*
978 * Insert the symbols into the hash table.
979 */
980 for (i = 0; i < mp->nsyms; i++) {
981 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
982
983 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
984 continue;
985 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
986 if (sp->st_shndx == SHN_COMMON)
987 sp->st_shndx = SHN_ABS;
988 }
989 sym_insert(mp, mp->strings + sp->st_name, i);
990 }
991
992 }
993
994 static kobj_notify_list_t **
995 kobj_notify_lookup(uint_t type)
996 {
997 ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
998 sizeof (kobj_notify_list_t *));
999
1000 return (&kobj_notifiers[type]);
1001 }
1002
1003 int
1004 kobj_notify_add(kobj_notify_list_t *knp)
1005 {
1006 kobj_notify_list_t **knl;
1007
1008 knl = kobj_notify_lookup(knp->kn_type);
1009
1010 knp->kn_next = NULL;
1011 knp->kn_prev = NULL;
1012
1013 mutex_enter(&kobj_lock);
1014
1015 if (*knl != NULL) {
1016 (*knl)->kn_prev = knp;
1017 knp->kn_next = *knl;
1018 }
1019 (*knl) = knp;
1020
1021 mutex_exit(&kobj_lock);
1022 return (0);
1023 }
1024
1025 int
1026 kobj_notify_remove(kobj_notify_list_t *knp)
1027 {
1028 kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
1029 kobj_notify_list_t *tknp;
1030
1031 mutex_enter(&kobj_lock);
1032
1033 /* LINTED */
1034 if (tknp = knp->kn_next)
1035 tknp->kn_prev = knp->kn_prev;
1036
1037 /* LINTED */
1038 if (tknp = knp->kn_prev)
1039 tknp->kn_next = knp->kn_next;
1040 else
1041 *knl = knp->kn_next;
1042
1043 mutex_exit(&kobj_lock);
1044
1045 return (0);
1046 }
1047
1048 /*
1049 * Notify all interested callbacks of a specified change in module state.
1050 */
1051 static void
1052 kobj_notify(int type, struct modctl *modp)
1053 {
1054 kobj_notify_list_t *knp;
1055
1056 if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1057 return;
1058
1059 mutex_enter(&kobj_lock);
1060
1061 for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1062 knp->kn_func(type, modp);
1063
1064 /*
1065 * KDI notification must be last (it has to allow for work done by the
1066 * other notification callbacks), so we call it manually.
1067 */
1068 kobj_kdi_mod_notify(type, modp);
1069
1070 mutex_exit(&kobj_lock);
1071 }
1072
1073 /*
1074 * Create the module path.
1075 */
1076 static char *
1077 getmodpath(const char *filename)
1078 {
1079 char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1080
1081 /*
1082 * Platform code gets first crack, then add
1083 * the default components
1084 */
1085 mach_modpath(path, filename);
1086 if (*path != '\0')
1087 (void) strcat(path, " ");
1088 return (strcat(path, MOD_DEFPATH));
1089 }
1090
1091 static struct modctl *
1092 add_primary(const char *filename, int lmid)
1093 {
1094 struct modctl *cp;
1095
1096 cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1097
1098 cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1099
1100 /*
1101 * For symbol lookup, we assemble our own
1102 * modctl list of the primary modules.
1103 */
1104
1105 (void) strcpy(cp->mod_filename, filename);
1106 cp->mod_modname = basename(cp->mod_filename);
1107
1108 /* set values for modinfo assuming that the load will work */
1109 cp->mod_prim = 1;
1110 cp->mod_loaded = 1;
1111 cp->mod_installed = 1;
1112 cp->mod_loadcnt = 1;
1113 cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1114
1115 cp->mod_id = kobj_last_module_id++;
1116
1117 /*
1118 * Link the module in. We'll pass this info on
1119 * to the mod squad later.
1120 */
1121 if (kobj_modules == NULL) {
1122 kobj_modules = cp;
1123 cp->mod_prev = cp->mod_next = cp;
1124 } else {
1125 cp->mod_prev = kobj_modules->mod_prev;
1126 cp->mod_next = kobj_modules;
1127 kobj_modules->mod_prev->mod_next = cp;
1128 kobj_modules->mod_prev = cp;
1129 }
1130
1131 kobj_lm_append(lmid, cp);
1132
1133 return (cp);
1134 }
1135
1136 static int
1137 bind_primary(val_t *bootaux, int lmid)
1138 {
1139 struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1140 struct modctl_list *lp;
1141 struct module *mp;
1142
1143 /*
1144 * Do common symbols.
1145 */
1146 for (lp = linkmap; lp; lp = lp->modl_next) {
1147 mp = mod(lp);
1148
1149 /*
1150 * Don't do common section relocations for modules that
1151 * don't need it.
1152 */
1153 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1154 continue;
1155
1156 if (do_common(mp) < 0)
1157 return (-1);
1158 }
1159
1160 /*
1161 * Resolve symbols.
1162 */
1163 for (lp = linkmap; lp; lp = lp->modl_next) {
1164 mp = mod(lp);
1165
1166 if (do_symbols(mp, 0) < 0)
1167 return (-1);
1168 }
1169
1170 /*
1171 * Do relocations.
1172 */
1173 for (lp = linkmap; lp; lp = lp->modl_next) {
1174 mp = mod(lp);
1175
1176 if (mp->flags & KOBJ_EXEC) {
1177 Dyn *dyn;
1178 Word relasz = 0, relaent = 0;
1179 char *rela = NULL;
1180
1181 for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1182 dyn->d_tag != DT_NULL; dyn++) {
1183 switch (dyn->d_tag) {
1184 case DT_RELASZ:
1185 case DT_RELSZ:
1186 relasz = dyn->d_un.d_val;
1187 break;
1188 case DT_RELAENT:
1189 case DT_RELENT:
1190 relaent = dyn->d_un.d_val;
1191 break;
1192 case DT_RELA:
1193 rela = (char *)dyn->d_un.d_ptr;
1194 break;
1195 case DT_REL:
1196 rela = (char *)dyn->d_un.d_ptr;
1197 break;
1198 }
1199 }
1200 if (relasz == 0 ||
1201 relaent == 0 || rela == NULL) {
1202 _kobj_printf(ops, "krtld: bind_primary(): "
1203 "no relocation information found for "
1204 "module %s\n", mp->filename);
1205 return (-1);
1206 }
1207 #ifdef KOBJ_DEBUG
1208 if (kobj_debug & D_RELOCATIONS)
1209 _kobj_printf(ops, "krtld: relocating: file=%s "
1210 "KOBJ_EXEC\n", mp->filename);
1211 #endif
1212 if (do_relocate(mp, rela, relasz/relaent, relaent,
1213 (Addr)mp->text) < 0)
1214 return (-1);
1215 } else {
1216 if (do_relocations(mp) < 0)
1217 return (-1);
1218 }
1219
1220 kobj_sync_instruction_memory(mp->text, mp->text_size);
1221 }
1222
1223 for (lp = linkmap; lp; lp = lp->modl_next) {
1224 mp = mod(lp);
1225
1226 /*
1227 * We need to re-read the full symbol table for the boot file,
1228 * since we couldn't use the full one before. We also need to
1229 * load the CTF sections of both the boot file and the
1230 * interpreter (us).
1231 */
1232 if (mp->flags & KOBJ_EXEC) {
1233 struct _buf *file;
1234 int n;
1235
1236 file = kobj_open_file(mp->filename);
1237 if (file == (struct _buf *)-1)
1238 return (-1);
1239 if (kobj_read_file(file, (char *)&mp->hdr,
1240 sizeof (mp->hdr), 0) < 0)
1241 return (-1);
1242 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1243 mp->shdrs = kobj_alloc(n, KM_WAIT);
1244 if (kobj_read_file(file, mp->shdrs, n,
1245 mp->hdr.e_shoff) < 0)
1246 return (-1);
1247 if (get_syms(mp, file) < 0)
1248 return (-1);
1249 if (get_ctf(mp, file) < 0)
1250 return (-1);
1251 kobj_close_file(file);
1252 mp->flags |= KOBJ_RELOCATED;
1253
1254 } else if (mp->flags & KOBJ_INTERP) {
1255 struct _buf *file;
1256
1257 /*
1258 * The interpreter path fragment in mp->filename
1259 * will already have the module directory suffix
1260 * in it (if appropriate).
1261 */
1262 file = kobj_open_path(mp->filename, 1, 0);
1263 if (file == (struct _buf *)-1)
1264 return (-1);
1265 if (get_ctf(mp, file) < 0)
1266 return (-1);
1267 kobj_close_file(file);
1268 mp->flags |= KOBJ_RELOCATED;
1269 }
1270 }
1271
1272 return (0);
1273 }
1274
1275 static struct modctl *
1276 mod_already_loaded(char *modname)
1277 {
1278 struct modctl *mctl = kobj_modules;
1279
1280 do {
1281 if (strcmp(modname, mctl->mod_filename) == 0)
1282 return (mctl);
1283 mctl = mctl->mod_next;
1284
1285 } while (mctl != kobj_modules);
1286
1287 return (NULL);
1288 }
1289
1290 /*
1291 * Load all the primary dependent modules.
1292 */
1293 static int
1294 load_primary(struct module *mp, int lmid)
1295 {
1296 struct modctl *cp;
1297 struct module *dmp;
1298 char *p, *q;
1299 char modname[MODMAXNAMELEN];
1300
1301 if ((p = mp->depends_on) == NULL)
1302 return (0);
1303
1304 /* CONSTANTCONDITION */
1305 while (1) {
1306 /*
1307 * Skip space.
1308 */
1309 while (*p && (*p == ' ' || *p == '\t'))
1310 p++;
1311 /*
1312 * Get module name.
1313 */
1314 q = modname;
1315 while (*p && *p != ' ' && *p != '\t')
1316 *q++ = *p++;
1317
1318 if (q == modname)
1319 break;
1320
1321 *q = '\0';
1322 /*
1323 * Check for dup dependencies.
1324 */
1325 if (strcmp(modname, "dtracestubs") == 0 ||
1326 mod_already_loaded(modname) != NULL)
1327 continue;
1328
1329 cp = add_primary(modname, lmid);
1330 cp->mod_busy = 1;
1331 /*
1332 * Load it.
1333 */
1334 (void) kobj_load_module(cp, 1);
1335 cp->mod_busy = 0;
1336
1337 if ((dmp = cp->mod_mp) == NULL) {
1338 cp->mod_loaded = 0;
1339 cp->mod_installed = 0;
1340 cp->mod_loadcnt = 0;
1341 return (-1);
1342 }
1343
1344 add_dependent(mp, dmp);
1345 dmp->flags |= KOBJ_PRIM;
1346
1347 /*
1348 * Recurse.
1349 */
1350 if (load_primary(dmp, lmid) == -1) {
1351 cp->mod_loaded = 0;
1352 cp->mod_installed = 0;
1353 cp->mod_loadcnt = 0;
1354 return (-1);
1355 }
1356 }
1357 return (0);
1358 }
1359
1360 static int
1361 console_is_usb_serial(void)
1362 {
1363 char *console;
1364 int len, ret;
1365
1366 if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1367 return (0);
1368
1369 console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1370 (void) BOP_GETPROP(ops, "console", console);
1371 ret = (strcmp(console, "usb-serial") == 0);
1372 kobj_free(console, len);
1373
1374 return (ret);
1375 }
1376
1377 static int
1378 load_kmdb(val_t *bootaux)
1379 {
1380 struct modctl *mctl;
1381 struct module *mp;
1382 Sym *sym;
1383
1384 if (console_is_usb_serial()) {
1385 _kobj_printf(ops, "kmdb not loaded "
1386 "(unsupported on usb serial console)\n");
1387 return (0);
1388 }
1389
1390 _kobj_printf(ops, "Loading kmdb...\n");
1391
1392 if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1393 return (-1);
1394
1395 mctl->mod_busy = 1;
1396 (void) kobj_load_module(mctl, 1);
1397 mctl->mod_busy = 0;
1398
1399 if ((mp = mctl->mod_mp) == NULL)
1400 return (-1);
1401
1402 mp->flags |= KOBJ_PRIM;
1403
1404 if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1405 return (-1);
1406
1407 if (boothowto & RB_VERBOSE)
1408 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1409
1410 if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1411 return (-1);
1412
1413 if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1414 return (-1);
1415
1416 #ifdef KOBJ_DEBUG
1417 if (kobj_debug & D_DEBUG) {
1418 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1419 sym->st_value);
1420 _kobj_printf(ops, "\tops 0x%p\n", ops);
1421 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1422 }
1423 #endif
1424
1425 if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1426 (const char **)kobj_kmdb_argv) < 0)
1427 return (-1);
1428
1429 return (0);
1430 }
1431
1432 /*
1433 * Return a string listing module dependencies.
1434 */
1435 static char *
1436 depends_on(struct module *mp)
1437 {
1438 Sym *sp;
1439 char *depstr, *q;
1440
1441 /*
1442 * The module doesn't have a depends_on value, so let's try it the
1443 * old-fashioned way - via "_depends_on"
1444 */
1445 if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1446 return (NULL);
1447
1448 q = (char *)sp->st_value;
1449
1450 #ifdef KOBJ_DEBUG
1451 /*
1452 * _depends_on is a deprecated interface, so we warn about its use
1453 * irrespective of subsequent processing errors. How else are we going
1454 * to be able to deco this interface completely?
1455 * Changes initially limited to DEBUG because third-party modules
1456 * should be flagged to developers before general use base.
1457 */
1458 _kobj_printf(ops,
1459 "Warning: %s uses deprecated _depends_on interface.\n",
1460 mp->filename);
1461 _kobj_printf(ops, "Please notify module developer or vendor.\n");
1462 #endif
1463
1464 /*
1465 * Idiot checks. Make sure it's
1466 * in-bounds and NULL terminated.
1467 */
1468 if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1469 _kobj_printf(ops, "Error processing dependency for %s\n",
1470 mp->filename);
1471 return (NULL);
1472 }
1473
1474 depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1475 (void) strcpy(depstr, q);
1476
1477 return (depstr);
1478 }
1479
1480 void
1481 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1482 {
1483 struct module *mp;
1484 mp = (struct module *)xmp;
1485
1486 modinfo->mi_base = mp->text;
1487 modinfo->mi_size = mp->text_size + mp->data_size;
1488 }
1489
1490 /*
1491 * kobj_export_ksyms() performs the following services:
1492 *
1493 * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1494 * (2) Removes unneeded symbols to save space.
1495 * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1496 * (4) Makes the symbol table visible to /dev/ksyms.
1497 */
1498 static void
1499 kobj_export_ksyms(struct module *mp)
1500 {
1501 Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1502 Sym *sp, *osp;
1503 char *name;
1504 size_t namelen;
1505 struct module *omp;
1506 uint_t nsyms;
1507 size_t symsize = mp->symhdr->sh_entsize;
1508 size_t locals = 1;
1509 size_t strsize;
1510
1511 /*
1512 * Make a copy of the original module structure.
1513 */
1514 omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1515 bcopy(mp, omp, sizeof (struct module));
1516
1517 /*
1518 * Compute the sizes of the new symbol table sections.
1519 */
1520 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1521 if (osp->st_value == 0)
1522 continue;
1523 if (sym_lookup(omp, osp) == NULL)
1524 continue;
1525 name = omp->strings + osp->st_name;
1526 namelen = strlen(name);
1527 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1528 locals++;
1529 nsyms++;
1530 strsize += namelen + 1;
1531 }
1532
1533 mp->nsyms = nsyms;
1534 mp->hashsize = kobj_gethashsize(mp->nsyms);
1535
1536 /*
1537 * ksyms_lock must be held as writer during any operation that
1538 * modifies ksyms_arena, including allocation from same, and
1539 * must not be dropped until the arena is vmem_walk()able.
1540 */
1541 rw_enter(&ksyms_lock, RW_WRITER);
1542
1543 /*
1544 * Allocate space for the new section headers (symtab and strtab),
1545 * symbol table, buckets, chains, and strings.
1546 */
1547 mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1548 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1549
1550 if (mp->flags & KOBJ_NOKSYMS) {
1551 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1552 } else {
1553 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1554 VM_BESTFIT | VM_SLEEP);
1555 }
1556 bzero(mp->symspace, mp->symsize);
1557
1558 /*
1559 * Divvy up symspace.
1560 */
1561 mp->shdrs = mp->symspace;
1562 mp->symhdr = (Shdr *)mp->shdrs;
1563 mp->strhdr = (Shdr *)(mp->symhdr + 1);
1564 mp->symtbl = (char *)(mp->strhdr + 1);
1565 mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1566 mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1567 mp->strings = (char *)(mp->chains + nsyms);
1568
1569 /*
1570 * Fill in the new section headers (symtab and strtab).
1571 */
1572 mp->hdr.e_shnum = 2;
1573 mp->symtbl_section = 0;
1574
1575 mp->symhdr->sh_type = SHT_SYMTAB;
1576 mp->symhdr->sh_addr = (Addr)mp->symtbl;
1577 mp->symhdr->sh_size = nsyms * symsize;
1578 mp->symhdr->sh_link = 1;
1579 mp->symhdr->sh_info = locals;
1580 mp->symhdr->sh_addralign = sizeof (Addr);
1581 mp->symhdr->sh_entsize = symsize;
1582
1583 mp->strhdr->sh_type = SHT_STRTAB;
1584 mp->strhdr->sh_addr = (Addr)mp->strings;
1585 mp->strhdr->sh_size = strsize;
1586 mp->strhdr->sh_addralign = 1;
1587
1588 /*
1589 * Construct the new symbol table.
1590 */
1591 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1592 if (osp->st_value == 0)
1593 continue;
1594 if (sym_lookup(omp, osp) == NULL)
1595 continue;
1596 name = omp->strings + osp->st_name;
1597 namelen = strlen(name);
1598 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1599 bcopy(osp, sp, symsize);
1600 bcopy(name, mp->strings + strsize, namelen);
1601 sp->st_name = strsize;
1602 sym_insert(mp, name, nsyms);
1603 nsyms++;
1604 strsize += namelen + 1;
1605 }
1606
1607 rw_exit(&ksyms_lock);
1608
1609 /*
1610 * Free the old section headers -- we'll never need them again.
1611 */
1612 if (!(mp->flags & KOBJ_PRIM)) {
1613 uint_t shn;
1614 Shdr *shp;
1615
1616 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1617 shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1618 switch (shp->sh_type) {
1619 case SHT_RELA:
1620 case SHT_REL:
1621 if (shp->sh_addr != 0) {
1622 kobj_free((void *)shp->sh_addr,
1623 shp->sh_size);
1624 }
1625 break;
1626 }
1627 }
1628 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1629 }
1630 /*
1631 * Discard the old symbol table and our copy of the module strucure.
1632 */
1633 if (!(mp->flags & KOBJ_PRIM))
1634 kobj_free(omp->symspace, omp->symsize);
1635 kobj_free(omp, sizeof (struct module));
1636 }
1637
1638 static void
1639 kobj_export_ctf(struct module *mp)
1640 {
1641 char *data = mp->ctfdata;
1642 size_t size = mp->ctfsize;
1643
1644 if (data != NULL) {
1645 if (_moddebug & MODDEBUG_NOCTF) {
1646 mp->ctfdata = NULL;
1647 mp->ctfsize = 0;
1648 } else {
1649 mp->ctfdata = vmem_alloc(ctf_arena, size,
1650 VM_BESTFIT | VM_SLEEP);
1651 bcopy(data, mp->ctfdata, size);
1652 }
1653
1654 if (!(mp->flags & KOBJ_PRIM))
1655 kobj_free(data, size);
1656 }
1657 }
1658
1659 void
1660 kobj_export_module(struct module *mp)
1661 {
1662 kobj_export_ksyms(mp);
1663 kobj_export_ctf(mp);
1664
1665 mp->flags |= KOBJ_EXPORTED;
1666 }
1667
1668 static int
1669 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1670 {
1671 char *path = NULL, *depstr = NULL;
1672 int allocsize = 0, osize = 0, nsize = 0;
1673 char *libname, *tmp;
1674 int lsize;
1675 Dyn *dynp;
1676
1677 for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1678 switch (dynp->d_tag) {
1679 case DT_NEEDED:
1680 /*
1681 * Read the DT_NEEDED entries, expanding the macros they
1682 * contain (if any), and concatenating them into a
1683 * single space-separated dependency list.
1684 */
1685 libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1686
1687 if (strchr(libname, '$') != NULL) {
1688 char *_lib;
1689
1690 if (path == NULL)
1691 path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1692 if ((_lib = expand_libmacro(libname, path,
1693 path)) != NULL)
1694 libname = _lib;
1695 else {
1696 _kobj_printf(ops, "krtld: "
1697 "process_dynamic: failed to expand "
1698 "%s\n", libname);
1699 }
1700 }
1701
1702 lsize = strlen(libname);
1703 nsize += lsize;
1704 if (nsize + 1 > allocsize) {
1705 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1706 KM_WAIT);
1707 if (depstr != NULL) {
1708 bcopy(depstr, tmp, osize);
1709 kobj_free(depstr, allocsize);
1710 }
1711 depstr = tmp;
1712 allocsize += MAXPATHLEN;
1713 }
1714 bcopy(libname, depstr + osize, lsize);
1715 *(depstr + nsize) = ' '; /* separator */
1716 nsize++;
1717 osize = nsize;
1718 break;
1719
1720 case DT_FLAGS_1:
1721 if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1722 mp->flags |= KOBJ_IGNMULDEF;
1723 if (dynp->d_un.d_val & DF_1_NOKSYMS)
1724 mp->flags |= KOBJ_NOKSYMS;
1725
1726 break;
1727 }
1728 }
1729
1730 /*
1731 * finish up the depends string (if any)
1732 */
1733 if (depstr != NULL) {
1734 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1735 if (path != NULL)
1736 kobj_free(path, MAXPATHLEN);
1737
1738 tmp = kobj_alloc(nsize, KM_WAIT);
1739 bcopy(depstr, tmp, nsize);
1740 kobj_free(depstr, allocsize);
1741 depstr = tmp;
1742
1743 mp->depends_on = depstr;
1744 }
1745
1746 return (0);
1747 }
1748
1749 static int
1750 do_dynamic(struct module *mp, struct _buf *file)
1751 {
1752 Shdr *dshp, *dstrp, *shp;
1753 char *dyndata, *dstrdata;
1754 int dshn, shn, rc;
1755
1756 /* find and validate the dynamic section (if any) */
1757
1758 for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1759 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1760 switch (shp->sh_type) {
1761 case SHT_DYNAMIC:
1762 if (dshp != NULL) {
1763 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1764 mp->filename);
1765 _kobj_printf(ops,
1766 "multiple dynamic sections\n");
1767 return (-1);
1768 } else {
1769 dshp = shp;
1770 dshn = shn;
1771 }
1772 break;
1773 }
1774 }
1775
1776 if (dshp == NULL)
1777 return (0);
1778
1779 if (dshp->sh_link > mp->hdr.e_shnum) {
1780 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1781 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1782 return (-1);
1783 }
1784 dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1785
1786 if (dstrp->sh_type != SHT_STRTAB) {
1787 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1788 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1789 dshn);
1790 return (-1);
1791 }
1792
1793 /* read it from disk */
1794
1795 dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1796 if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1797 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1798 _kobj_printf(ops, "error reading section %d\n", dshn);
1799
1800 kobj_free(dyndata, dshp->sh_size);
1801 return (-1);
1802 }
1803
1804 dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1805 if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1806 dstrp->sh_offset) < 0) {
1807 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1808 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1809
1810 kobj_free(dyndata, dshp->sh_size);
1811 kobj_free(dstrdata, dstrp->sh_size);
1812 return (-1);
1813 }
1814
1815 /* pull the interesting pieces out */
1816
1817 rc = process_dynamic(mp, dyndata, dstrdata);
1818
1819 kobj_free(dyndata, dshp->sh_size);
1820 kobj_free(dstrdata, dstrp->sh_size);
1821
1822 return (rc);
1823 }
1824
1825 void
1826 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1827 {
1828 if (!standalone) {
1829 if (mp->ctfdata != NULL) {
1830 if (vmem_contains(ctf_arena, mp->ctfdata,
1831 mp->ctfsize)) {
1832 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1833 } else {
1834 kobj_free(mp->ctfdata, mp->ctfsize);
1835 }
1836 }
1837 }
1838
1839 /*
1840 * The order is very important here. We need to make sure that
1841 * consumers, at any given instant, see a consistent state. We'd
1842 * rather they see no CTF data than the address of one buffer and the
1843 * size of another.
1844 */
1845 mp->ctfdata = NULL;
1846 membar_producer();
1847 mp->ctfsize = size;
1848 mp->ctfdata = data;
1849 membar_producer();
1850 }
1851
1852 int
1853 kobj_load_module(struct modctl *modp, int use_path)
1854 {
1855 char *filename = modp->mod_filename;
1856 char *modname = modp->mod_modname;
1857 int i;
1858 int n;
1859 struct _buf *file;
1860 struct module *mp = NULL;
1861 #ifdef MODDIR_SUFFIX
1862 int no_suffixdir_drv = 0;
1863 #endif
1864
1865 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1866
1867 /*
1868 * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1869 * kmdb contains a bunch of symbols with well-known names, symbols
1870 * which will mask the real versions, thus causing no end of trouble
1871 * for mdb.
1872 */
1873 if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1874 mp->flags |= KOBJ_NOKSYMS;
1875
1876 file = kobj_open_path(filename, use_path, 1);
1877 if (file == (struct _buf *)-1) {
1878 #ifdef MODDIR_SUFFIX
1879 file = kobj_open_path(filename, use_path, 0);
1880 #endif
1881 if (file == (struct _buf *)-1) {
1882 kobj_free(mp, sizeof (*mp));
1883 goto bad;
1884 }
1885 #ifdef MODDIR_SUFFIX
1886 /*
1887 * There is no driver module in the ISA specific (suffix)
1888 * subdirectory but there is a module in the parent directory.
1889 */
1890 if (strncmp(filename, "drv/", 4) == 0) {
1891 no_suffixdir_drv = 1;
1892 }
1893 #endif
1894 }
1895
1896 mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1897 (void) strcpy(mp->filename, file->_name);
1898
1899 if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1900 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1901 modname);
1902 kobj_free(mp->filename, strlen(file->_name) + 1);
1903 kobj_free(mp, sizeof (*mp));
1904 goto bad;
1905 }
1906 for (i = 0; i < SELFMAG; i++) {
1907 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1908 if (_moddebug & MODDEBUG_ERRMSG)
1909 _kobj_printf(ops, "%s not an elf module\n",
1910 modname);
1911 kobj_free(mp->filename, strlen(file->_name) + 1);
1912 kobj_free(mp, sizeof (*mp));
1913 goto bad;
1914 }
1915 }
1916 /*
1917 * It's ELF, but is it our ISA? Interpreting the header
1918 * from a file for a byte-swapped ISA could cause a huge
1919 * and unsatisfiable value to be passed to kobj_alloc below
1920 * and therefore hang booting.
1921 */
1922 if (!elf_mach_ok(&mp->hdr)) {
1923 if (_moddebug & MODDEBUG_ERRMSG)
1924 _kobj_printf(ops, "%s not an elf module for this ISA\n",
1925 modname);
1926 kobj_free(mp->filename, strlen(file->_name) + 1);
1927 kobj_free(mp, sizeof (*mp));
1928 #ifdef MODDIR_SUFFIX
1929 /*
1930 * The driver mod is not in the ISA specific subdirectory
1931 * and the module in the parent directory is not our ISA.
1932 * If it is our ISA, for now we will silently succeed.
1933 */
1934 if (no_suffixdir_drv == 1) {
1935 cmn_err(CE_CONT, "?NOTICE: %s: 64-bit driver module"
1936 " not found\n", modname);
1937 }
1938 #endif
1939 goto bad;
1940 }
1941
1942 /*
1943 * All modules, save for unix, should be relocatable (as opposed to
1944 * dynamic). Dynamic modules come with PLTs and GOTs, which can't
1945 * currently be processed by krtld.
1946 */
1947 if (mp->hdr.e_type != ET_REL) {
1948 if (_moddebug & MODDEBUG_ERRMSG)
1949 _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1950 "module\n", modname);
1951 kobj_free(mp->filename, strlen(file->_name) + 1);
1952 kobj_free(mp, sizeof (*mp));
1953 goto bad;
1954 }
1955
1956 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1957 mp->shdrs = kobj_alloc(n, KM_WAIT);
1958
1959 if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1960 _kobj_printf(ops, "kobj_load_module: %s error reading "
1961 "section headers\n", modname);
1962 kobj_free(mp->shdrs, n);
1963 kobj_free(mp->filename, strlen(file->_name) + 1);
1964 kobj_free(mp, sizeof (*mp));
1965 goto bad;
1966 }
1967
1968 kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1969 module_assign(modp, mp);
1970
1971 /* read in sections */
1972 if (get_progbits(mp, file) < 0) {
1973 _kobj_printf(ops, "%s error reading sections\n", modname);
1974 goto bad;
1975 }
1976
1977 if (do_dynamic(mp, file) < 0) {
1978 _kobj_printf(ops, "%s error reading dynamic section\n",
1979 modname);
1980 goto bad;
1981 }
1982
1983 modp->mod_text = mp->text;
1984 modp->mod_text_size = mp->text_size;
1985
1986 /* read in symbols; adjust values for each section's real address */
1987 if (get_syms(mp, file) < 0) {
1988 _kobj_printf(ops, "%s error reading symbols\n",
1989 modname);
1990 goto bad;
1991 }
1992
1993 /*
1994 * If we didn't dependency information from the dynamic section, look
1995 * for it the old-fashioned way.
1996 */
1997 if (mp->depends_on == NULL)
1998 mp->depends_on = depends_on(mp);
1999
2000 if (get_ctf(mp, file) < 0) {
2001 _kobj_printf(ops, "%s debug information will not "
2002 "be available\n", modname);
2003 }
2004
2005 /* primary kernel modules do not have a signature section */
2006 if (!(mp->flags & KOBJ_PRIM))
2007 get_signature(mp, file);
2008
2009 #ifdef KOBJ_DEBUG
2010 if (kobj_debug & D_LOADING) {
2011 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
2012 _kobj_printf(ops, "\ttext:0x%p", mp->text);
2013 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
2014 _kobj_printf(ops, "\tdata:0x%p", mp->data);
2015 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
2016 }
2017 #endif /* KOBJ_DEBUG */
2018
2019 /*
2020 * For primary kernel modules, we defer
2021 * symbol resolution and relocation until
2022 * all primary objects have been loaded.
2023 */
2024 if (!standalone) {
2025 int ddrval, dcrval;
2026 char *dependent_modname;
2027 /* load all dependents */
2028 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
2029 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
2030
2031 /*
2032 * resolve undefined and common symbols,
2033 * also allocates common space
2034 */
2035 if ((dcrval = do_common(mp)) < 0) {
2036 switch (dcrval) {
2037 case DOSYM_UNSAFE:
2038 _kobj_printf(ops, "WARNING: mod_load: "
2039 "MT-unsafe module '%s' rejected\n",
2040 modname);
2041 break;
2042 case DOSYM_UNDEF:
2043 _kobj_printf(ops, "WARNING: mod_load: "
2044 "cannot load module '%s'\n",
2045 modname);
2046 if (ddrval == -1) {
2047 _kobj_printf(ops, "WARNING: %s: ",
2048 modname);
2049 _kobj_printf(ops,
2050 "unable to resolve dependency, "
2051 "module '%s' not found\n",
2052 dependent_modname);
2053 }
2054 break;
2055 }
2056 }
2057 kobj_free(dependent_modname, MODMAXNAMELEN);
2058 if (dcrval < 0)
2059 goto bad;
2060
2061 /* process relocation tables */
2062 if (do_relocations(mp) < 0) {
2063 _kobj_printf(ops, "%s error doing relocations\n",
2064 modname);
2065 goto bad;
2066 }
2067
2068 if (mp->destination) {
2069 off_t off = (uintptr_t)mp->destination & PAGEOFFSET;
2070 caddr_t base = (caddr_t)mp->destination - off;
2071 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2072
2073 hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2074 vmem_free(heap_arena, base, size);
2075 }
2076
2077 /* sync_instruction_memory */
2078 kobj_sync_instruction_memory(mp->text, mp->text_size);
2079 kobj_export_module(mp);
2080 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2081 }
2082 kobj_close_file(file);
2083 return (0);
2084 bad:
2085 if (file != (struct _buf *)-1)
2086 kobj_close_file(file);
2087 if (modp->mod_mp != NULL)
2088 free_module_data(modp->mod_mp);
2089
2090 module_assign(modp, NULL);
2091 return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2092 }
2093
2094 int
2095 kobj_load_primary_module(struct modctl *modp)
2096 {
2097 struct modctl *dep;
2098 struct module *mp;
2099
2100 if (kobj_load_module(modp, 0) != 0)
2101 return (-1);
2102
2103 dep = NULL;
2104 mp = modp->mod_mp;
2105 mp->flags |= KOBJ_PRIM;
2106
2107 /* Bind new module to its dependents */
2108 if (mp->depends_on != NULL && (dep =
2109 mod_already_loaded(mp->depends_on)) == NULL) {
2110 #ifdef KOBJ_DEBUG
2111 if (kobj_debug & D_DEBUG) {
2112 _kobj_printf(ops, "krtld: failed to resolve deps "
2113 "for primary %s\n", modp->mod_modname);
2114 }
2115 #endif
2116 return (-1);
2117 }
2118
2119 if (dep != NULL)
2120 add_dependent(mp, dep->mod_mp);
2121
2122 /*
2123 * Relocate it. This module may not be part of a link map, so we
2124 * can't use bind_primary.
2125 */
2126 if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2127 do_relocations(mp) < 0) {
2128 #ifdef KOBJ_DEBUG
2129 if (kobj_debug & D_DEBUG) {
2130 _kobj_printf(ops, "krtld: failed to relocate "
2131 "primary %s\n", modp->mod_modname);
2132 }
2133 #endif
2134 return (-1);
2135 }
2136
2137 return (0);
2138 }
2139
2140 static void
2141 module_assign(struct modctl *cp, struct module *mp)
2142 {
2143 if (standalone) {
2144 cp->mod_mp = mp;
2145 return;
2146 }
2147 mutex_enter(&mod_lock);
2148 cp->mod_mp = mp;
2149 cp->mod_gencount++;
2150 mutex_exit(&mod_lock);
2151 }
2152
2153 void
2154 kobj_unload_module(struct modctl *modp)
2155 {
2156 struct module *mp = modp->mod_mp;
2157
2158 if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2159 _kobj_printf(ops, "text for %s ", mp->filename);
2160 _kobj_printf(ops, "was at %p\n", mp->text);
2161 mp->text = NULL; /* don't actually free it */
2162 }
2163
2164 kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2165
2166 /*
2167 * Null out mod_mp first, so consumers (debuggers) know not to look
2168 * at the module structure any more.
2169 */
2170 mutex_enter(&mod_lock);
2171 modp->mod_mp = NULL;
2172 mutex_exit(&mod_lock);
2173
2174 kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2175 free_module_data(mp);
2176 }
2177
2178 static void
2179 free_module_data(struct module *mp)
2180 {
2181 struct module_list *lp, *tmp;
2182 hotinline_desc_t *hid, *next;
2183 int ksyms_exported = 0;
2184
2185 lp = mp->head;
2186 while (lp) {
2187 tmp = lp;
2188 lp = lp->next;
2189 kobj_free((char *)tmp, sizeof (*tmp));
2190 }
2191
2192 /* release hotinlines */
2193 hid = mp->hi_calls;
2194 while (hid != NULL) {
2195 next = hid->hid_next;
2196 kobj_free(hid->hid_symname, strlen(hid->hid_symname) + 1);
2197 kobj_free(hid, sizeof (hotinline_desc_t));
2198 hid = next;
2199 }
2200
2201 rw_enter(&ksyms_lock, RW_WRITER);
2202 if (mp->symspace) {
2203 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2204 vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2205 ksyms_exported = 1;
2206 } else {
2207 if (mp->flags & KOBJ_NOKSYMS)
2208 ksyms_exported = 1;
2209 kobj_free(mp->symspace, mp->symsize);
2210 }
2211 }
2212 rw_exit(&ksyms_lock);
2213
2214 if (mp->ctfdata) {
2215 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2216 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2217 else
2218 kobj_free(mp->ctfdata, mp->ctfsize);
2219 }
2220
2221 if (mp->sigdata)
2222 kobj_free(mp->sigdata, mp->sigsize);
2223
2224 /*
2225 * We did not get far enough into kobj_export_ksyms() to free allocated
2226 * buffers because we encounted error conditions. Free the buffers.
2227 */
2228 if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2229 uint_t shn;
2230 Shdr *shp;
2231
2232 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2233 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2234 switch (shp->sh_type) {
2235 case SHT_RELA:
2236 case SHT_REL:
2237 if (shp->sh_addr != 0)
2238 kobj_free((void *)shp->sh_addr,
2239 shp->sh_size);
2240 break;
2241 }
2242 }
2243 err_free_done:
2244 if (!(mp->flags & KOBJ_PRIM)) {
2245 kobj_free(mp->shdrs,
2246 mp->hdr.e_shentsize * mp->hdr.e_shnum);
2247 }
2248 }
2249
2250 if (mp->bss)
2251 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2252
2253 if (mp->fbt_tab)
2254 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2255
2256 if (mp->textwin_base)
2257 kobj_textwin_free(mp);
2258
2259 if (mp->sdt_probes != NULL) {
2260 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2261
2262 while (sdp != NULL) {
2263 next = sdp->sdpd_next;
2264 kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2265 kobj_free(sdp, sizeof (sdt_probedesc_t));
2266 sdp = next;
2267 }
2268 }
2269
2270 if (mp->sdt_tab)
2271 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2272 if (mp->text)
2273 vmem_free(text_arena, mp->text, mp->text_size);
2274 if (mp->data)
2275 vmem_free(data_arena, mp->data, mp->data_size);
2276 if (mp->depends_on)
2277 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2278 if (mp->filename)
2279 kobj_free(mp->filename, strlen(mp->filename)+1);
2280
2281 kobj_free((char *)mp, sizeof (*mp));
2282 }
2283
2284 static int
2285 get_progbits(struct module *mp, struct _buf *file)
2286 {
2287 struct proginfo *tp, *dp, *sdp;
2288 Shdr *shp;
2289 reloc_dest_t dest = NULL;
2290 uintptr_t bits_ptr;
2291 uintptr_t text = 0, data, textptr;
2292 uint_t shn;
2293 int err = -1;
2294
2295 tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2296 dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2297 sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2298 /*
2299 * loop through sections to find out how much space we need
2300 * for text, data, (also bss that is already assigned)
2301 */
2302 if (get_progbits_size(mp, tp, dp, sdp) < 0)
2303 goto done;
2304
2305 mp->text_size = tp->size;
2306 mp->data_size = dp->size;
2307
2308 if (standalone) {
2309 caddr_t limit = _data;
2310
2311 if (lg_pagesize && _text + lg_pagesize < limit)
2312 limit = _text + lg_pagesize;
2313
2314 mp->text = kobj_segbrk(&_etext, mp->text_size,
2315 tp->align, limit);
2316 /*
2317 * If we can't grow the text segment, try the
2318 * data segment before failing.
2319 */
2320 if (mp->text == NULL) {
2321 mp->text = kobj_segbrk(&_edata, mp->text_size,
2322 tp->align, 0);
2323 }
2324
2325 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2326
2327 if (mp->text == NULL || mp->data == NULL)
2328 goto done;
2329
2330 } else {
2331 if (text_arena == NULL)
2332 kobj_vmem_init(&text_arena, &data_arena);
2333
2334 /*
2335 * some architectures may want to load the module on a
2336 * page that is currently read only. It may not be
2337 * possible for those architectures to remap their page
2338 * on the fly. So we provide a facility for them to hang
2339 * a private hook where the memory they assign the module
2340 * is not the actual place where the module loads.
2341 *
2342 * In this case there are two addresses that deal with the
2343 * modload.
2344 * 1) the final destination of the module
2345 * 2) the address that is used to view the newly
2346 * loaded module until all the relocations relative to 1
2347 * above are completed.
2348 *
2349 * That is what dest is used for below.
2350 */
2351 mp->text_size += tp->align;
2352 mp->data_size += dp->align;
2353
2354 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2355
2356 /*
2357 * a remap is taking place. Align the text ptr relative
2358 * to the secondary mapping. That is where the bits will
2359 * be read in.
2360 */
2361 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2362 mp->text, mp->text_size)) {
2363 off_t off = (uintptr_t)mp->text & PAGEOFFSET;
2364 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2365 caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2366 caddr_t orig = mp->text - off;
2367 pgcnt_t pages = size / PAGESIZE;
2368
2369 dest = (reloc_dest_t)(map + off);
2370 text = ALIGN((uintptr_t)dest, tp->align);
2371
2372 while (pages--) {
2373 hat_devload(kas.a_hat, map, PAGESIZE,
2374 hat_getpfnum(kas.a_hat, orig),
2375 PROT_READ | PROT_WRITE | PROT_EXEC,
2376 HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2377 map += PAGESIZE;
2378 orig += PAGESIZE;
2379 }
2380 /*
2381 * Since we set up a non-cacheable mapping, we need
2382 * to flush any old entries in the cache that might
2383 * be left around from the read-only mapping.
2384 */
2385 dcache_flushall();
2386 }
2387 if (mp->data_size)
2388 mp->data = vmem_alloc(data_arena, mp->data_size,
2389 VM_SLEEP | VM_BESTFIT);
2390 }
2391 textptr = (uintptr_t)mp->text;
2392 textptr = ALIGN(textptr, tp->align);
2393 mp->destination = dest;
2394
2395 /*
2396 * This is the case where a remap is not being done.
2397 */
2398 if (text == 0)
2399 text = ALIGN((uintptr_t)mp->text, tp->align);
2400 data = ALIGN((uintptr_t)mp->data, dp->align);
2401
2402 /* now loop though sections assigning addresses and loading the data */
2403 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2404 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2405 if (!(shp->sh_flags & SHF_ALLOC))
2406 continue;
2407
2408 if ((shp->sh_flags & SHF_WRITE) == 0)
2409 bits_ptr = text;
2410 else
2411 bits_ptr = data;
2412
2413 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2414
2415 if (shp->sh_type == SHT_NOBITS) {
2416 /*
2417 * Zero bss.
2418 */
2419 bzero((caddr_t)bits_ptr, shp->sh_size);
2420 shp->sh_type = SHT_PROGBITS;
2421 } else {
2422 if (kobj_read_file(file, (char *)bits_ptr,
2423 shp->sh_size, shp->sh_offset) < 0)
2424 goto done;
2425 }
2426
2427 if (shp->sh_flags & SHF_WRITE) {
2428 shp->sh_addr = bits_ptr;
2429 } else {
2430 textptr = ALIGN(textptr, shp->sh_addralign);
2431 shp->sh_addr = textptr;
2432 textptr += shp->sh_size;
2433 }
2434
2435 bits_ptr += shp->sh_size;
2436 if ((shp->sh_flags & SHF_WRITE) == 0)
2437 text = bits_ptr;
2438 else
2439 data = bits_ptr;
2440 }
2441
2442 err = 0;
2443 done:
2444 /*
2445 * Free and mark as freed the section headers here so that
2446 * free_module_data() does not have to worry about this buffer.
2447 *
2448 * This buffer is freed here because one of the possible reasons
2449 * for error is a section with non-zero sh_addr and in that case
2450 * free_module_data() would have no way of recognizing that this
2451 * buffer was unallocated.
2452 */
2453 if (err != 0) {
2454 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2455 mp->shdrs = NULL;
2456 }
2457
2458 (void) kobj_free(tp, sizeof (struct proginfo));
2459 (void) kobj_free(dp, sizeof (struct proginfo));
2460 (void) kobj_free(sdp, sizeof (struct proginfo));
2461
2462 return (err);
2463 }
2464
2465 /*
2466 * Go through suppress_sym_list to see if "multiply defined"
2467 * warning of this symbol should be suppressed. Return 1 if
2468 * warning should be suppressed, 0 otherwise.
2469 */
2470 static int
2471 kobj_suppress_warning(char *symname)
2472 {
2473 int i;
2474
2475 for (i = 0; suppress_sym_list[i] != NULL; i++) {
2476 if (strcmp(suppress_sym_list[i], symname) == 0)
2477 return (1);
2478 }
2479
2480 return (0);
2481 }
2482
2483 static int
2484 get_syms(struct module *mp, struct _buf *file)
2485 {
2486 uint_t shn;
2487 Shdr *shp;
2488 uint_t i;
2489 Sym *sp, *ksp;
2490 char *symname;
2491 int dosymtab = 0;
2492
2493 /*
2494 * Find the interesting sections.
2495 */
2496 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2497 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2498 switch (shp->sh_type) {
2499 case SHT_SYMTAB:
2500 mp->symtbl_section = shn;
2501 mp->symhdr = shp;
2502 dosymtab++;
2503 break;
2504
2505 case SHT_RELA:
2506 case SHT_REL:
2507 /*
2508 * Already loaded.
2509 */
2510 if (shp->sh_addr)
2511 continue;
2512
2513 /* KM_TMP since kobj_free'd in do_relocations */
2514 shp->sh_addr = (Addr)
2515 kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2516
2517 if (kobj_read_file(file, (char *)shp->sh_addr,
2518 shp->sh_size, shp->sh_offset) < 0) {
2519 _kobj_printf(ops, "krtld: get_syms: %s, ",
2520 mp->filename);
2521 _kobj_printf(ops, "error reading section %d\n",
2522 shn);
2523 return (-1);
2524 }
2525 break;
2526 }
2527 }
2528
2529 /*
2530 * This is true for a stripped executable. In the case of
2531 * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2532 * and since that symbol information is still present everything
2533 * is just fine.
2534 */
2535 if (!dosymtab) {
2536 if (mp->flags & KOBJ_EXEC)
2537 return (0);
2538 _kobj_printf(ops, "krtld: get_syms: %s ",
2539 mp->filename);
2540 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2541 return (-1);
2542 }
2543
2544 /*
2545 * get the associated string table header
2546 */
2547 if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2548 return (-1);
2549 mp->strhdr = (Shdr *)
2550 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2551
2552 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2553 mp->hashsize = kobj_gethashsize(mp->nsyms);
2554
2555 /*
2556 * Allocate space for the symbol table, buckets, chains, and strings.
2557 */
2558 mp->symsize = mp->symhdr->sh_size +
2559 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2560 mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2561
2562 mp->symtbl = mp->symspace;
2563 mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2564 mp->chains = mp->buckets + mp->hashsize;
2565 mp->strings = (char *)(mp->chains + mp->nsyms);
2566
2567 if (kobj_read_file(file, mp->symtbl,
2568 mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2569 kobj_read_file(file, mp->strings,
2570 mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2571 return (-1);
2572
2573 /*
2574 * loop through the symbol table adjusting values to account
2575 * for where each section got loaded into memory. Also
2576 * fill in the hash table.
2577 */
2578 for (i = 1; i < mp->nsyms; i++) {
2579 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2580 if (sp->st_shndx < SHN_LORESERVE) {
2581 if (sp->st_shndx >= mp->hdr.e_shnum) {
2582 _kobj_printf(ops, "%s bad shndx ",
2583 file->_name);
2584 _kobj_printf(ops, "in symbol %d\n", i);
2585 return (-1);
2586 }
2587 shp = (Shdr *)
2588 (mp->shdrs +
2589 sp->st_shndx * mp->hdr.e_shentsize);
2590 if (!(mp->flags & KOBJ_EXEC))
2591 sp->st_value += shp->sh_addr;
2592 }
2593
2594 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2595 continue;
2596 if (sp->st_name >= mp->strhdr->sh_size)
2597 return (-1);
2598
2599 symname = mp->strings + sp->st_name;
2600
2601 if (!(mp->flags & KOBJ_EXEC) &&
2602 ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2603 ksp = kobj_lookup_all(mp, symname, 0);
2604
2605 if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2606 !kobj_suppress_warning(symname) &&
2607 sp->st_shndx != SHN_UNDEF &&
2608 sp->st_shndx != SHN_COMMON &&
2609 ksp->st_shndx != SHN_UNDEF &&
2610 ksp->st_shndx != SHN_COMMON) {
2611 /*
2612 * Unless this symbol is a stub, it's multiply
2613 * defined. Multiply-defined symbols are
2614 * usually bad, but some objects (kmdb) have
2615 * a legitimate need to have their own
2616 * copies of common functions.
2617 */
2618 if ((standalone ||
2619 ksp->st_value < (uintptr_t)stubs_base ||
2620 ksp->st_value >= (uintptr_t)stubs_end) &&
2621 !(mp->flags & KOBJ_IGNMULDEF)) {
2622 _kobj_printf(ops,
2623 "%s symbol ", file->_name);
2624 _kobj_printf(ops,
2625 "%s multiply defined\n", symname);
2626 }
2627 }
2628 }
2629
2630 sym_insert(mp, symname, i);
2631 }
2632
2633 return (0);
2634 }
2635
2636 static int
2637 get_ctf(struct module *mp, struct _buf *file)
2638 {
2639 char *shstrtab, *ctfdata;
2640 size_t shstrlen;
2641 Shdr *shp;
2642 uint_t i;
2643
2644 if (_moddebug & MODDEBUG_NOCTF)
2645 return (0); /* do not attempt to even load CTF data */
2646
2647 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2648 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2649 mp->filename);
2650 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2651 mp->hdr.e_shstrndx);
2652 return (-1);
2653 }
2654
2655 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2656 shstrlen = shp->sh_size;
2657 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2658
2659 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2660 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2661 mp->filename);
2662 _kobj_printf(ops, "error reading section %u\n",
2663 mp->hdr.e_shstrndx);
2664 kobj_free(shstrtab, shstrlen);
2665 return (-1);
2666 }
2667
2668 for (i = 0; i < mp->hdr.e_shnum; i++) {
2669 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2670
2671 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2672 strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2673 ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2674
2675 if (kobj_read_file(file, ctfdata, shp->sh_size,
2676 shp->sh_offset) < 0) {
2677 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2678 "reading .SUNW_ctf data\n", mp->filename);
2679 kobj_free(ctfdata, shp->sh_size);
2680 kobj_free(shstrtab, shstrlen);
2681 return (-1);
2682 }
2683
2684 mp->ctfdata = ctfdata;
2685 mp->ctfsize = shp->sh_size;
2686 break;
2687 }
2688 }
2689
2690 kobj_free(shstrtab, shstrlen);
2691 return (0);
2692 }
2693
2694 #define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */
2695
2696 /*
2697 * Return the hash of the ELF sections that are memory resident.
2698 * i.e. text and data. We skip a SHT_NOBITS section since it occupies
2699 * no space in the file. We use SHA1 here since libelfsign uses
2700 * it and both places need to use the same algorithm.
2701 */
2702 static void
2703 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2704 {
2705 uint_t shn;
2706 Shdr *shp;
2707 SHA1_CTX ctx;
2708
2709 SHA1Init(&ctx);
2710
2711 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2712 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2713 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2714 continue;
2715
2716 /*
2717 * The check should ideally be shp->sh_type == SHT_NOBITS.
2718 * However, we can't do that check here as get_progbits()
2719 * resets the type.
2720 */
2721 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2722 continue;
2723 #ifdef KOBJ_DEBUG
2724 if (kobj_debug & D_DEBUG)
2725 _kobj_printf(ops,
2726 "krtld: crypto_es_hash: updating hash with"
2727 " %s data size=%lx\n", shstrtab + shp->sh_name,
2728 (size_t)shp->sh_size);
2729 #endif
2730 ASSERT(shp->sh_addr != 0);
2731 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2732 }
2733
2734 SHA1Final((uchar_t *)hash, &ctx);
2735 }
2736
2737 /*
2738 * Get the .SUNW_signature section for the module, it it exists.
2739 *
2740 * This section exists only for crypto modules. None of the
2741 * primary modules have this section currently.
2742 */
2743 static void
2744 get_signature(struct module *mp, struct _buf *file)
2745 {
2746 char *shstrtab, *sigdata = NULL;
2747 size_t shstrlen;
2748 Shdr *shp;
2749 uint_t i;
2750
2751 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2752 _kobj_printf(ops, "krtld: get_signature: %s, ",
2753 mp->filename);
2754 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2755 mp->hdr.e_shstrndx);
2756 return;
2757 }
2758
2759 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2760 shstrlen = shp->sh_size;
2761 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2762
2763 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2764 _kobj_printf(ops, "krtld: get_signature: %s, ",
2765 mp->filename);
2766 _kobj_printf(ops, "error reading section %u\n",
2767 mp->hdr.e_shstrndx);
2768 kobj_free(shstrtab, shstrlen);
2769 return;
2770 }
2771
2772 for (i = 0; i < mp->hdr.e_shnum; i++) {
2773 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2774 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2775 strcmp(shstrtab + shp->sh_name,
2776 ELF_SIGNATURE_SECTION) == 0) {
2777 filesig_vers_t filesig_version;
2778 size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2779 sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2780
2781 if (kobj_read_file(file, sigdata, shp->sh_size,
2782 shp->sh_offset) < 0) {
2783 _kobj_printf(ops, "krtld: get_signature: %s,"
2784 " error reading .SUNW_signature data\n",
2785 mp->filename);
2786 kobj_free(sigdata, sigsize);
2787 kobj_free(shstrtab, shstrlen);
2788 return;
2789 }
2790 filesig_version = ((struct filesignatures *)sigdata)->
2791 filesig_sig.filesig_version;
2792 if (!(filesig_version == FILESIG_VERSION1 ||
2793 filesig_version == FILESIG_VERSION3)) {
2794 /* skip versions we don't understand */
2795 kobj_free(sigdata, sigsize);
2796 kobj_free(shstrtab, shstrlen);
2797 return;
2798 }
2799
2800 mp->sigdata = sigdata;
2801 mp->sigsize = sigsize;
2802 break;
2803 }
2804 }
2805
2806 if (sigdata != NULL) {
2807 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2808 }
2809
2810 kobj_free(shstrtab, shstrlen);
2811 }
2812
2813 static void
2814 add_dependent(struct module *mp, struct module *dep)
2815 {
2816 struct module_list *lp;
2817
2818 for (lp = mp->head; lp; lp = lp->next) {
2819 if (lp->mp == dep)
2820 return; /* already on the list */
2821 }
2822
2823 if (lp == NULL) {
2824 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2825
2826 lp->mp = dep;
2827 lp->next = NULL;
2828 if (mp->tail)
2829 mp->tail->next = lp;
2830 else
2831 mp->head = lp;
2832 mp->tail = lp;
2833 }
2834 }
2835
2836 static int
2837 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2838 {
2839 struct module *mp;
2840 struct modctl *req;
2841 char *d, *p, *q;
2842 int c;
2843 char *err_modname = NULL;
2844
2845 mp = modp->mod_mp;
2846
2847 if ((p = mp->depends_on) == NULL)
2848 return (0);
2849
2850 for (;;) {
2851 /*
2852 * Skip space.
2853 */
2854 while (*p && (*p == ' ' || *p == '\t'))
2855 p++;
2856 /*
2857 * Get module name.
2858 */
2859 d = p;
2860 q = modname;
2861 c = 0;
2862 while (*p && *p != ' ' && *p != '\t') {
2863 if (c < modnamelen - 1) {
2864 *q++ = *p;
2865 c++;
2866 }
2867 p++;
2868 }
2869
2870 if (q == modname)
2871 break;
2872
2873 if (c == modnamelen - 1) {
2874 char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2875
2876 (void) strncpy(dep, d, p - d + 1);
2877 dep[p - d] = '\0';
2878
2879 _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2880 _kobj_printf(ops, "'%s' too long ", dep);
2881 _kobj_printf(ops, "(max %d chars)\n", (int)modnamelen);
2882
2883 kobj_free(dep, p - d + 1);
2884
2885 return (-1);
2886 }
2887
2888 *q = '\0';
2889 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2890 #ifndef KOBJ_DEBUG
2891 if (_moddebug & MODDEBUG_LOADMSG) {
2892 #endif /* KOBJ_DEBUG */
2893 _kobj_printf(ops,
2894 "%s: unable to resolve dependency, ",
2895 modp->mod_modname);
2896 _kobj_printf(ops, "cannot load module '%s'\n",
2897 modname);
2898 #ifndef KOBJ_DEBUG
2899 }
2900 #endif /* KOBJ_DEBUG */
2901 if (err_modname == NULL) {
2902 /*
2903 * This must be the same size as the modname
2904 * one.
2905 */
2906 err_modname = kobj_zalloc(MODMAXNAMELEN,
2907 KM_WAIT);
2908
2909 /*
2910 * We can use strcpy() here without fearing
2911 * the NULL terminator because the size of
2912 * err_modname is the same as one of modname,
2913 * and it's filled with zeros.
2914 */
2915 (void) strcpy(err_modname, modname);
2916 }
2917 continue;
2918 }
2919
2920 add_dependent(mp, req->mod_mp);
2921 mod_release_mod(req);
2922
2923 }
2924
2925 if (err_modname != NULL) {
2926 /*
2927 * Copy the first module name where you detect an error to keep
2928 * its behavior the same as before.
2929 * This way keeps minimizing the memory use for error
2930 * modules, and this might be important at boot time because
2931 * the memory usage is a crucial factor for booting in most
2932 * cases. You can expect more verbose messages when using
2933 * a debug kernel or setting a bit in moddebug.
2934 */
2935 bzero(modname, MODMAXNAMELEN);
2936 (void) strcpy(modname, err_modname);
2937 kobj_free(err_modname, MODMAXNAMELEN);
2938 return (-1);
2939 }
2940
2941 return (0);
2942 }
2943
2944 static int
2945 do_common(struct module *mp)
2946 {
2947 int err;
2948
2949 /*
2950 * first time through, assign all symbols defined in other
2951 * modules, and count up how much common space will be needed
2952 * (bss_size and bss_align)
2953 */
2954 if ((err = do_symbols(mp, 0)) < 0)
2955 return (err);
2956 /*
2957 * increase bss_size by the maximum delta that could be
2958 * computed by the ALIGN below
2959 */
2960 mp->bss_size += mp->bss_align;
2961 if (mp->bss_size) {
2962 if (standalone)
2963 mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2964 MINALIGN, 0);
2965 else
2966 mp->bss = (uintptr_t)vmem_alloc(data_arena,
2967 mp->bss_size, VM_SLEEP | VM_BESTFIT);
2968 bzero((void *)mp->bss, mp->bss_size);
2969 /* now assign addresses to all common symbols */
2970 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2971 return (err);
2972 }
2973 return (0);
2974 }
2975
2976 static int
2977 do_symbols(struct module *mp, Elf64_Addr bss_base)
2978 {
2979 int bss_align;
2980 uintptr_t bss_ptr;
2981 int err;
2982 int i;
2983 Sym *sp, *sp1;
2984 char *name;
2985 int assign;
2986 int resolved = 1;
2987
2988 /*
2989 * Nothing left to do (optimization).
2990 */
2991 if (mp->flags & KOBJ_RESOLVED)
2992 return (0);
2993
2994 assign = (bss_base) ? 1 : 0;
2995 bss_ptr = bss_base;
2996 bss_align = 0;
2997 err = 0;
2998
2999 for (i = 1; i < mp->nsyms; i++) {
3000 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
3001 /*
3002 * we know that st_name is in bounds, since get_sections
3003 * has already checked all of the symbols
3004 */
3005 name = mp->strings + sp->st_name;
3006 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
3007 continue;
3008 #if defined(__sparc)
3009 /*
3010 * Register symbols are ignored in the kernel
3011 */
3012 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER) {
3013 if (*name != '\0') {
3014 _kobj_printf(ops, "%s: named REGISTER symbol ",
3015 mp->filename);
3016 _kobj_printf(ops, "not supported '%s'\n",
3017 name);
3018 err = DOSYM_UNDEF;
3019 }
3020 continue;
3021 }
3022 #endif /* __sparc */
3023 /*
3024 * TLS symbols are ignored in the kernel
3025 */
3026 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
3027 _kobj_printf(ops, "%s: TLS symbol ",
3028 mp->filename);
3029 _kobj_printf(ops, "not supported '%s'\n",
3030 name);
3031 err = DOSYM_UNDEF;
3032 continue;
3033 }
3034
3035 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
3036 if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
3037 sp->st_shndx = SHN_ABS;
3038 sp->st_value = sp1->st_value;
3039 continue;
3040 }
3041 }
3042
3043 if (sp->st_shndx == SHN_UNDEF) {
3044 resolved = 0;
3045
3046 /*
3047 * Skip over sdt probes and smap calls,
3048 * they're relocated later.
3049 */
3050 if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
3051 continue;
3052 #if defined(__x86)
3053 if (strcmp(name, "smap_enable") == 0 ||
3054 strcmp(name, "smap_disable") == 0)
3055 continue;
3056 #endif /* defined(__x86) */
3057
3058
3059 /*
3060 * If it's not a weak reference and it's
3061 * not a primary object, it's an error.
3062 * (Primary objects may take more than
3063 * one pass to resolve)
3064 */
3065 if (!(mp->flags & KOBJ_PRIM) &&
3066 ELF_ST_BIND(sp->st_info) != STB_WEAK) {
3067 _kobj_printf(ops, "%s: undefined symbol",
3068 mp->filename);
3069 _kobj_printf(ops, " '%s'\n", name);
3070 /*
3071 * Try to determine whether this symbol
3072 * represents a dependency on obsolete
3073 * unsafe driver support. This is just
3074 * to make the warning more informative.
3075 */
3076 if (strcmp(name, "sleep") == 0 ||
3077 strcmp(name, "unsleep") == 0 ||
3078 strcmp(name, "wakeup") == 0 ||
3079 strcmp(name, "bsd_compat_ioctl") == 0 ||
3080 strcmp(name, "unsafe_driver") == 0 ||
3081 strncmp(name, "spl", 3) == 0 ||
3082 strncmp(name, "i_ddi_spl", 9) == 0)
3083 err = DOSYM_UNSAFE;
3084 if (err == 0)
3085 err = DOSYM_UNDEF;
3086 }
3087 continue;
3088 }
3089 /*
3090 * It's a common symbol - st_value is the
3091 * required alignment.
3092 */
3093 if (sp->st_value > bss_align)
3094 bss_align = sp->st_value;
3095 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3096 if (assign) {
3097 sp->st_shndx = SHN_ABS;
3098 sp->st_value = bss_ptr;
3099 }
3100 bss_ptr += sp->st_size;
3101 }
3102 if (err)
3103 return (err);
3104 if (assign == 0 && mp->bss == 0) {
3105 mp->bss_align = bss_align;
3106 mp->bss_size = bss_ptr;
3107 } else if (resolved) {
3108 mp->flags |= KOBJ_RESOLVED;
3109 }
3110
3111 return (0);
3112 }
3113
3114 uint_t
3115 kobj_hash_name(const char *p)
3116 {
3117 uint_t g;
3118 uint_t hval;
3119
3120 hval = 0;
3121 while (*p) {
3122 hval = (hval << 4) + *p++;
3123 if ((g = (hval & 0xf0000000)) != 0)
3124 hval ^= g >> 24;
3125 hval &= ~g;
3126 }
3127 return (hval);
3128 }
3129
3130 /* look for name in all modules */
3131 uintptr_t
3132 kobj_getsymvalue(char *name, int kernelonly)
3133 {
3134 Sym *sp;
3135 struct modctl *modp;
3136 struct module *mp;
3137 uintptr_t value = 0;
3138
3139 if ((sp = kobj_lookup_kernel(name)) != NULL)
3140 return ((uintptr_t)sp->st_value);
3141
3142 if (kernelonly)
3143 return (0); /* didn't find it in the kernel so give up */
3144
3145 mutex_enter(&mod_lock);
3146 modp = &modules;
3147 do {
3148 mp = (struct module *)modp->mod_mp;
3149 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3150 (sp = lookup_one(mp, name))) {
3151 value = (uintptr_t)sp->st_value;
3152 break;
3153 }
3154 } while ((modp = modp->mod_next) != &modules);
3155 mutex_exit(&mod_lock);
3156 return (value);
3157 }
3158
3159 /* look for a symbol near value. */
3160 char *
3161 kobj_getsymname(uintptr_t value, ulong_t *offset)
3162 {
3163 char *name = NULL;
3164 struct modctl *modp;
3165
3166 struct modctl_list *lp;
3167 struct module *mp;
3168
3169 /*
3170 * Loop through the primary kernel modules.
3171 */
3172 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3173 mp = mod(lp);
3174
3175 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3176 return (name);
3177 }
3178
3179 mutex_enter(&mod_lock);
3180 modp = &modules;
3181 do {
3182 mp = (struct module *)modp->mod_mp;
3183 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3184 (name = kobj_searchsym(mp, value, offset)))
3185 break;
3186 } while ((modp = modp->mod_next) != &modules);
3187 mutex_exit(&mod_lock);
3188 return (name);
3189 }
3190
3191 /* return address of symbol and size */
3192
3193 uintptr_t
3194 kobj_getelfsym(char *name, void *mp, int *size)
3195 {
3196 Sym *sp;
3197
3198 if (mp == NULL)
3199 sp = kobj_lookup_kernel(name);
3200 else
3201 sp = lookup_one(mp, name);
3202
3203 if (sp == NULL)
3204 return (0);
3205
3206 *size = (int)sp->st_size;
3207 return ((uintptr_t)sp->st_value);
3208 }
3209
3210 uintptr_t
3211 kobj_lookup(struct module *mod, const char *name)
3212 {
3213 Sym *sp;
3214
3215 sp = lookup_one(mod, name);
3216
3217 if (sp == NULL)
3218 return (0);
3219
3220 return ((uintptr_t)sp->st_value);
3221 }
3222
3223 char *
3224 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3225 {
3226 Sym *symtabptr;
3227 char *strtabptr;
3228 int symnum;
3229 Sym *sym;
3230 Sym *cursym;
3231 uintptr_t curval;
3232
3233 *offset = (ulong_t)-1l; /* assume not found */
3234 cursym = NULL;
3235
3236 if (kobj_addrcheck(mp, (void *)value) != 0)
3237 return (NULL); /* not in this module */
3238
3239 strtabptr = mp->strings;
3240 symtabptr = (Sym *)mp->symtbl;
3241
3242 /*
3243 * Scan the module's symbol table for a symbol <= value
3244 */
3245 for (symnum = 1, sym = symtabptr + 1;
3246 symnum < mp->nsyms; symnum++, sym = (Sym *)
3247 ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3248 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3249 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3250 continue;
3251 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3252 ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3253 continue;
3254 }
3255
3256 curval = (uintptr_t)sym->st_value;
3257
3258 if (curval > value)
3259 continue;
3260
3261 /*
3262 * If one or both are functions...
3263 */
3264 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3265 ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3266 /* Ignore if the address is out of the bounds */
3267 if (value - sym->st_value >= sym->st_size)
3268 continue;
3269
3270 if (cursym != NULL &&
3271 ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3272 /* Prefer the function to the non-function */
3273 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3274 continue;
3275
3276 /* Prefer the larger of the two functions */
3277 if (sym->st_size <= cursym->st_size)
3278 continue;
3279 }
3280 } else if (value - curval >= *offset) {
3281 continue;
3282 }
3283
3284 *offset = (ulong_t)(value - curval);
3285 cursym = sym;
3286 }
3287 if (cursym == NULL)
3288 return (NULL);
3289
3290 return (strtabptr + cursym->st_name);
3291 }
3292
3293 Sym *
3294 kobj_lookup_all(struct module *mp, char *name, int include_self)
3295 {
3296 Sym *sp;
3297 struct module_list *mlp;
3298 struct modctl_list *clp;
3299 struct module *mmp;
3300
3301 if (include_self && (sp = lookup_one(mp, name)) != NULL)
3302 return (sp);
3303
3304 for (mlp = mp->head; mlp; mlp = mlp->next) {
3305 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3306 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3307 return (sp);
3308 }
3309
3310 /*
3311 * Loop through the primary kernel modules.
3312 */
3313 for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3314 mmp = mod(clp);
3315
3316 if (mmp == NULL || mp == mmp)
3317 continue;
3318
3319 if ((sp = lookup_one(mmp, name)) != NULL &&
3320 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3321 return (sp);
3322 }
3323 return (NULL);
3324 }
3325
3326 Sym *
3327 kobj_lookup_kernel(const char *name)
3328 {
3329 struct modctl_list *lp;
3330 struct module *mp;
3331 Sym *sp;
3332
3333 /*
3334 * Loop through the primary kernel modules.
3335 */
3336 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3337 mp = mod(lp);
3338
3339 if (mp == NULL)
3340 continue;
3341
3342 if ((sp = lookup_one(mp, name)) != NULL)
3343 return (sp);
3344 }
3345 return (NULL);
3346 }
3347
3348 static Sym *
3349 lookup_one(struct module *mp, const char *name)
3350 {
3351 symid_t *ip;
3352 char *name1;
3353 Sym *sp;
3354
3355 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3356 ip = &mp->chains[*ip]) {
3357 sp = (Sym *)(mp->symtbl +
3358 mp->symhdr->sh_entsize * *ip);
3359 name1 = mp->strings + sp->st_name;
3360 if (strcmp(name, name1) == 0 &&
3361 ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3362 sp->st_shndx != SHN_UNDEF &&
3363 sp->st_shndx != SHN_COMMON)
3364 return (sp);
3365 }
3366 return (NULL);
3367 }
3368
3369 /*
3370 * Lookup a given symbol pointer in the module's symbol hash. If the symbol
3371 * is hashed, return the symbol pointer; otherwise return NULL.
3372 */
3373 static Sym *
3374 sym_lookup(struct module *mp, Sym *ksp)
3375 {
3376 char *name = mp->strings + ksp->st_name;
3377 symid_t *ip;
3378 Sym *sp;
3379
3380 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3381 ip = &mp->chains[*ip]) {
3382 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3383 if (sp == ksp)
3384 return (ksp);
3385 }
3386 return (NULL);
3387 }
3388
3389 static void
3390 sym_insert(struct module *mp, char *name, symid_t index)
3391 {
3392 symid_t *ip;
3393
3394 #ifdef KOBJ_DEBUG
3395 if (kobj_debug & D_SYMBOLS) {
3396 static struct module *lastmp = NULL;
3397 Sym *sp;
3398 if (lastmp != mp) {
3399 _kobj_printf(ops,
3400 "krtld: symbol entry: file=%s\n",
3401 mp->filename);
3402 _kobj_printf(ops,
3403 "krtld:\tsymndx\tvalue\t\t"
3404 "symbol name\n");
3405 lastmp = mp;
3406 }
3407 sp = (Sym *)(mp->symtbl +
3408 index * mp->symhdr->sh_entsize);
3409 _kobj_printf(ops, "krtld:\t[%3d]", index);
3410 _kobj_printf(ops, "\t0x%lx", sp->st_value);
3411 _kobj_printf(ops, "\t%s\n", name);
3412 }
3413 #endif
3414
3415 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3416 ip = &mp->chains[*ip]) {
3417 ;
3418 }
3419 *ip = index;
3420 }
3421
3422 struct modctl *
3423 kobj_boot_mod_lookup(const char *modname)
3424 {
3425 struct modctl *mctl = kobj_modules;
3426
3427 do {
3428 if (strcmp(modname, mctl->mod_modname) == 0)
3429 return (mctl);
3430 } while ((mctl = mctl->mod_next) != kobj_modules);
3431
3432 return (NULL);
3433 }
3434
3435 /*
3436 * Determine if the module exists.
3437 */
3438 int
3439 kobj_path_exists(char *name, int use_path)
3440 {
3441 struct _buf *file;
3442
3443 file = kobj_open_path(name, use_path, 1);
3444 #ifdef MODDIR_SUFFIX
3445 if (file == (struct _buf *)-1)
3446 file = kobj_open_path(name, use_path, 0);
3447 #endif /* MODDIR_SUFFIX */
3448 if (file == (struct _buf *)-1)
3449 return (0);
3450 kobj_close_file(file);
3451 return (1);
3452 }
3453
3454 /*
3455 * fullname is dynamically allocated to be able to hold the
3456 * maximum size string that can be constructed from name.
3457 * path is exactly like the shell PATH variable.
3458 */
3459 struct _buf *
3460 kobj_open_path(char *name, int use_path, int use_moddir_suffix)
3461 {
3462 char *p, *q;
3463 char *pathp;
3464 char *pathpsave;
3465 char *fullname;
3466 int maxpathlen;
3467 struct _buf *file;
3468
3469 #if !defined(MODDIR_SUFFIX)
3470 use_moddir_suffix = B_FALSE;
3471 #endif
3472
3473 if (!use_path)
3474 pathp = ""; /* use name as specified */
3475 else
3476 pathp = kobj_module_path;
3477 /* use configured default path */
3478
3479 pathpsave = pathp; /* keep this for error reporting */
3480
3481 /*
3482 * Allocate enough space for the largest possible fullname.
3483 * since path is of the form <directory> : <directory> : ...
3484 * we're potentially allocating a little more than we need to
3485 * but we'll allocate the exact amount when we find the right directory.
3486 * (The + 3 below is one for NULL terminator and one for the '/'
3487 * we might have to add at the beginning of path and one for
3488 * the '/' between path and name.)
3489 */
3490 maxpathlen = strlen(pathp) + strlen(name) + 3;
3491 /* sizeof includes null */
3492 maxpathlen += sizeof (slash_moddir_suffix_slash) - 1;
3493 fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3494
3495 for (;;) {
3496 p = fullname;
3497 if (*pathp != '\0' && *pathp != '/')
3498 *p++ = '/'; /* path must start with '/' */
3499 while (*pathp && *pathp != ':' && *pathp != ' ')
3500 *p++ = *pathp++;
3501 if (p != fullname && p[-1] != '/')
3502 *p++ = '/';
3503 if (use_moddir_suffix) {
3504 char *b = basename(name);
3505 char *s;
3506
3507 /* copy everything up to the base name */
3508 q = name;
3509 while (q != b && *q)
3510 *p++ = *q++;
3511 s = slash_moddir_suffix_slash;
3512 while (*s)
3513 *p++ = *s++;
3514 /* copy the rest */
3515 while (*b)
3516 *p++ = *b++;
3517 } else {
3518 q = name;
3519 while (*q)
3520 *p++ = *q++;
3521 }
3522 *p = 0;
3523 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3524 kobj_free(fullname, maxpathlen);
3525 return (file);
3526 }
3527 while (*pathp == ' ' || *pathp == ':')
3528 pathp++;
3529 if (*pathp == 0)
3530 break;
3531
3532 }
3533 kobj_free(fullname, maxpathlen);
3534 if (_moddebug & MODDEBUG_ERRMSG) {
3535 _kobj_printf(ops, "can't open %s,", name);
3536 _kobj_printf(ops, " path is %s\n", pathpsave);
3537 }
3538 return ((struct _buf *)-1);
3539 }
3540
3541 intptr_t
3542 kobj_open(char *filename)
3543 {
3544 struct vnode *vp;
3545 int fd;
3546
3547 if (_modrootloaded) {
3548 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3549 int Errno;
3550
3551 /*
3552 * Hand off the open to a thread who has a
3553 * stack size capable handling the request.
3554 */
3555 if (curthread != &t0) {
3556 (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3557 kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3558 sema_p(<p->sema);
3559 Errno = ltp->Errno;
3560 vp = ltp->vp;
3561 } else {
3562 /*
3563 * 1098067: module creds should not be those of the
3564 * caller
3565 */
3566 cred_t *saved_cred = curthread->t_cred;
3567 curthread->t_cred = kcred;
3568 Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3569 0, 0, rootdir, -1);
3570 curthread->t_cred = saved_cred;
3571 }
3572 kobjopen_free(ltp);
3573
3574 if (Errno) {
3575 if (_moddebug & MODDEBUG_ERRMSG) {
3576 _kobj_printf(ops,
3577 "kobj_open: vn_open of %s fails, ",
3578 filename);
3579 _kobj_printf(ops, "Errno = %d\n", Errno);
3580 }
3581 return (-1);
3582 } else {
3583 if (_moddebug & MODDEBUG_ERRMSG) {
3584 _kobj_printf(ops, "kobj_open: '%s'", filename);
3585 _kobj_printf(ops, " vp = %p\n", vp);
3586 }
3587 return ((intptr_t)vp);
3588 }
3589 } else {
3590 fd = kobj_boot_open(filename, 0);
3591
3592 if (_moddebug & MODDEBUG_ERRMSG) {
3593 if (fd < 0)
3594 _kobj_printf(ops,
3595 "kobj_open: can't open %s\n", filename);
3596 else {
3597 _kobj_printf(ops, "kobj_open: '%s'", filename);
3598 _kobj_printf(ops, " descr = 0x%x\n", fd);
3599 }
3600 }
3601 return ((intptr_t)fd);
3602 }
3603 }
3604
3605 /*
3606 * Calls to kobj_open() are handled off to this routine as a separate thread.
3607 */
3608 static void
3609 kobjopen_thread(struct kobjopen_tctl *ltp)
3610 {
3611 kmutex_t cpr_lk;
3612 callb_cpr_t cpr_i;
3613
3614 mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3615 CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3616 ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3617 0, 0);
3618 sema_v(<p->sema);
3619 mutex_enter(&cpr_lk);
3620 CALLB_CPR_EXIT(&cpr_i);
3621 mutex_destroy(&cpr_lk);
3622 thread_exit();
3623 }
3624
3625 /*
3626 * allocate and initialize a kobjopen thread structure
3627 */
3628 static struct kobjopen_tctl *
3629 kobjopen_alloc(char *filename)
3630 {
3631 struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3632
3633 ASSERT(filename != NULL);
3634
3635 ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3636 bcopy(filename, ltp->name, strlen(filename) + 1);
3637 sema_init(<p->sema, 0, NULL, SEMA_DEFAULT, NULL);
3638 return (ltp);
3639 }
3640
3641 /*
3642 * free a kobjopen thread control structure
3643 */
3644 static void
3645 kobjopen_free(struct kobjopen_tctl *ltp)
3646 {
3647 sema_destroy(<p->sema);
3648 kmem_free(ltp->name, strlen(ltp->name) + 1);
3649 kmem_free(ltp, sizeof (*ltp));
3650 }
3651
3652 int
3653 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3654 {
3655 int stat;
3656 ssize_t resid;
3657
3658 if (_modrootloaded) {
3659 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3660 (offset_t)offset, UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3661 &resid)) != 0) {
3662 _kobj_printf(ops,
3663 "vn_rdwr failed with error 0x%x\n", stat);
3664 return (-1);
3665 }
3666 return (size - resid);
3667 } else {
3668 int count = 0;
3669
3670 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3671 _kobj_printf(ops,
3672 "kobj_read: seek 0x%x failed\n", offset);
3673 return (-1);
3674 }
3675
3676 count = kobj_boot_read((int)descr, buf, size);
3677 if (count < size) {
3678 if (_moddebug & MODDEBUG_ERRMSG) {
3679 _kobj_printf(ops,
3680 "kobj_read: req %d bytes, ", size);
3681 _kobj_printf(ops, "got %d\n", count);
3682 }
3683 }
3684 return (count);
3685 }
3686 }
3687
3688 void
3689 kobj_close(intptr_t descr)
3690 {
3691 if (_moddebug & MODDEBUG_ERRMSG)
3692 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3693
3694 if (_modrootloaded) {
3695 struct vnode *vp = (struct vnode *)descr;
3696 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
3697 VN_RELE(vp);
3698 } else
3699 (void) kobj_boot_close((int)descr);
3700 }
3701
3702 int
3703 kobj_fstat(intptr_t descr, struct bootstat *buf)
3704 {
3705 if (buf == NULL)
3706 return (-1);
3707
3708 if (_modrootloaded) {
3709 vattr_t vattr;
3710 struct vnode *vp = (struct vnode *)descr;
3711 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3712 return (-1);
3713
3714 /*
3715 * The vattr and bootstat structures are similar, but not
3716 * identical. We do our best to fill in the bootstat structure
3717 * from the contents of vattr (transfering only the ones that
3718 * are obvious.
3719 */
3720
3721 buf->st_mode = (uint32_t)vattr.va_mode;
3722 buf->st_nlink = (uint32_t)vattr.va_nlink;
3723 buf->st_uid = (int32_t)vattr.va_uid;
3724 buf->st_gid = (int32_t)vattr.va_gid;
3725 buf->st_rdev = (uint64_t)vattr.va_rdev;
3726 buf->st_size = (uint64_t)vattr.va_size;
3727 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3728 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3729 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3730 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3731 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3732 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3733 buf->st_blksize = (int32_t)vattr.va_blksize;
3734 buf->st_blocks = (int64_t)vattr.va_nblocks;
3735
3736 return (0);
3737 }
3738
3739 return (kobj_boot_fstat((int)descr, buf));
3740 }
3741
3742
3743 struct _buf *
3744 kobj_open_file(char *name)
3745 {
3746 struct _buf *file;
3747 struct compinfo cbuf;
3748 intptr_t fd;
3749
3750 if ((fd = kobj_open(name)) == -1) {
3751 return ((struct _buf *)-1);
3752 }
3753
3754 file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3755 file->_fd = fd;
3756 file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3757 file->_cnt = file->_size = file->_off = 0;
3758 file->_ln = 1;
3759 file->_ptr = file->_base;
3760 (void) strcpy(file->_name, name);
3761
3762 /*
3763 * Before root is mounted, we must check
3764 * for a compressed file and do our own
3765 * buffering.
3766 */
3767 if (_modrootloaded) {
3768 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3769 file->_bsize = MAXBSIZE;
3770
3771 /* Check if the file is compressed */
3772 file->_iscmp = kobj_is_compressed(fd);
3773 } else {
3774 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3775 kobj_close_file(file);
3776 return ((struct _buf *)-1);
3777 }
3778 file->_iscmp = cbuf.iscmp;
3779 if (file->_iscmp) {
3780 if (kobj_comp_setup(file, &cbuf) != 0) {
3781 kobj_close_file(file);
3782 return ((struct _buf *)-1);
3783 }
3784 } else {
3785 file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3786 file->_bsize = cbuf.blksize;
3787 }
3788 }
3789 return (file);
3790 }
3791
3792 static int
3793 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3794 {
3795 struct comphdr *hdr;
3796
3797 /*
3798 * read the compressed image into memory,
3799 * so we can deompress from there
3800 */
3801 file->_dsize = cip->fsize;
3802 file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3803 if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3804 kobj_free(file->_dbuf, cip->fsize);
3805 return (-1);
3806 }
3807
3808 hdr = kobj_comphdr(file);
3809 if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3810 hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3811 !ISP2(hdr->ch_blksize)) {
3812 kobj_free(file->_dbuf, cip->fsize);
3813 return (-1);
3814 }
3815 file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3816 file->_bsize = hdr->ch_blksize;
3817 return (0);
3818 }
3819
3820 void
3821 kobj_close_file(struct _buf *file)
3822 {
3823 kobj_close(file->_fd);
3824 if (file->_base != NULL)
3825 kobj_free(file->_base, file->_bsize);
3826 if (file->_dbuf != NULL)
3827 kobj_free(file->_dbuf, file->_dsize);
3828 kobj_free(file->_name, strlen(file->_name)+1);
3829 kobj_free(file, sizeof (struct _buf));
3830 }
3831
3832 int
3833 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3834 {
3835 int b_size, c_size;
3836 int b_off; /* Offset into buffer for start of bcopy */
3837 int count = 0;
3838 int page_addr;
3839
3840 if (_moddebug & MODDEBUG_ERRMSG) {
3841 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3842 _kobj_printf(ops, " offset=%x at", off);
3843 _kobj_printf(ops, " buf=%lx\n", (uintptr_t)buf);
3844 }
3845
3846 /*
3847 * Handle compressed (gzip for now) file here. First get the
3848 * compressed size, then read the image into memory and finally
3849 * call zlib to decompress the image at the supplied memory buffer.
3850 */
3851 if (file->_iscmp == CH_MAGIC_GZIP) {
3852 ulong_t dlen;
3853 vattr_t vattr;
3854 struct vnode *vp = (struct vnode *)file->_fd;
3855 ssize_t resid;
3856 int err = 0;
3857
3858 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3859 return (-1);
3860
3861 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3862 file->_dsize = vattr.va_size;
3863
3864 /* Read the compressed file into memory */
3865 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3866 (offset_t)(0), UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3867 &resid)) != 0) {
3868
3869 _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3870 "error code 0x%x\n", err);
3871 return (-1);
3872 }
3873
3874 dlen = size;
3875
3876 /* Decompress the image at the supplied memory buffer */
3877 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3878 vattr.va_size)) != Z_OK) {
3879 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3880 "failed, error code : 0x%x\n", err);
3881 return (-1);
3882 }
3883
3884 if (dlen != size) {
3885 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3886 "failed to uncompress (size returned 0x%lx , "
3887 "expected size: 0x%x)\n", dlen, size);
3888 return (-1);
3889 }
3890
3891 return (0);
3892 }
3893
3894 while (size) {
3895 page_addr = F_PAGE(file, off);
3896 b_size = file->_size;
3897 /*
3898 * If we have the filesystem page the caller's referring to
3899 * and we have something in the buffer,
3900 * satisfy as much of the request from the buffer as we can.
3901 */
3902 if (page_addr == file->_off && b_size > 0) {
3903 b_off = B_OFFSET(file, off);
3904 c_size = b_size - b_off;
3905 /*
3906 * If there's nothing to copy, we're at EOF.
3907 */
3908 if (c_size <= 0)
3909 break;
3910 if (c_size > size)
3911 c_size = size;
3912 if (buf) {
3913 if (_moddebug & MODDEBUG_ERRMSG)
3914 _kobj_printf(ops, "copying %x bytes\n",
3915 c_size);
3916 bcopy(file->_base+b_off, buf, c_size);
3917 size -= c_size;
3918 off += c_size;
3919 buf += c_size;
3920 count += c_size;
3921 } else {
3922 _kobj_printf(ops, "kobj_read: system error");
3923 count = -1;
3924 break;
3925 }
3926 } else {
3927 /*
3928 * If the caller's offset is page aligned and
3929 * the caller want's at least a filesystem page and
3930 * the caller provided a buffer,
3931 * read directly into the caller's buffer.
3932 */
3933 if (page_addr == off &&
3934 (c_size = F_BLKS(file, size)) && buf) {
3935 c_size = kobj_read_blks(file, buf, c_size,
3936 page_addr);
3937 if (c_size < 0) {
3938 count = -1;
3939 break;
3940 }
3941 count += c_size;
3942 if (c_size != F_BLKS(file, size))
3943 break;
3944 size -= c_size;
3945 off += c_size;
3946 buf += c_size;
3947 /*
3948 * Otherwise, read into our buffer and copy next time
3949 * around the loop.
3950 */
3951 } else {
3952 file->_off = page_addr;
3953 c_size = kobj_read_blks(file, file->_base,
3954 file->_bsize, page_addr);
3955 file->_ptr = file->_base;
3956 file->_cnt = c_size;
3957 file->_size = c_size;
3958 /*
3959 * If a _filbuf call or nothing read, break.
3960 */
3961 if (buf == NULL || c_size <= 0) {
3962 count = c_size;
3963 break;
3964 }
3965 }
3966 if (_moddebug & MODDEBUG_ERRMSG)
3967 _kobj_printf(ops, "read %x bytes\n", c_size);
3968 }
3969 }
3970 if (_moddebug & MODDEBUG_ERRMSG)
3971 _kobj_printf(ops, "count = %x\n", count);
3972
3973 return (count);
3974 }
3975
3976 static int
3977 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3978 {
3979 int ret;
3980
3981 ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3982 if (file->_iscmp) {
3983 uint_t blks;
3984 int nret;
3985
3986 ret = 0;
3987 for (blks = size / file->_bsize; blks != 0; blks--) {
3988 nret = kobj_uncomp_blk(file, buf, off);
3989 if (nret == -1)
3990 return (-1);
3991 buf += nret;
3992 off += nret;
3993 ret += nret;
3994 if (nret < file->_bsize)
3995 break;
3996 }
3997 } else
3998 ret = kobj_read(file->_fd, buf, size, off);
3999 return (ret);
4000 }
4001
4002 static int
4003 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
4004 {
4005 struct comphdr *hdr = kobj_comphdr(file);
4006 ulong_t dlen, slen;
4007 caddr_t src;
4008 int i;
4009
4010 dlen = file->_bsize;
4011 i = off / file->_bsize;
4012 src = file->_dbuf + hdr->ch_blkmap[i];
4013 if (i == hdr->ch_fsize / file->_bsize)
4014 slen = file->_dsize - hdr->ch_blkmap[i];
4015 else
4016 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
4017 if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
4018 return (-1);
4019 return (dlen);
4020 }
4021
4022 int
4023 kobj_filbuf(struct _buf *f)
4024 {
4025 if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
4026 return (kobj_getc(f));
4027 return (-1);
4028 }
4029
4030 void
4031 kobj_free(void *address, size_t size)
4032 {
4033 if (standalone)
4034 return;
4035
4036 kmem_free(address, size);
4037 kobj_stat.nfree_calls++;
4038 kobj_stat.nfree += size;
4039 }
4040
4041 void *
4042 kobj_zalloc(size_t size, int flag)
4043 {
4044 void *v;
4045
4046 if ((v = kobj_alloc(size, flag)) != 0) {
4047 bzero(v, size);
4048 }
4049
4050 return (v);
4051 }
4052
4053 void *
4054 kobj_alloc(size_t size, int flag)
4055 {
4056 /*
4057 * If we are running standalone in the
4058 * linker, we ask boot for memory.
4059 * Either it's temporary memory that we lose
4060 * once boot is mapped out or we allocate it
4061 * permanently using the dynamic data segment.
4062 */
4063 if (standalone) {
4064 #if defined(_OBP)
4065 if (flag & (KM_TMP | KM_SCRATCH))
4066 return (bop_temp_alloc(size, MINALIGN));
4067 #else
4068 if (flag & (KM_TMP | KM_SCRATCH))
4069 return (BOP_ALLOC(ops, 0, size, MINALIGN));
4070 #endif
4071 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
4072 }
4073
4074 kobj_stat.nalloc_calls++;
4075 kobj_stat.nalloc += size;
4076
4077 return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
4078 }
4079
4080 /*
4081 * Allow the "mod" system to sync up with the work
4082 * already done by kobj during the initial loading
4083 * of the kernel. This also gives us a chance
4084 * to reallocate memory that belongs to boot.
4085 */
4086 void
4087 kobj_sync(void)
4088 {
4089 struct modctl_list *lp, **lpp;
4090
4091 /*
4092 * The module path can be set in /etc/system via 'moddir' commands
4093 */
4094 if (default_path != NULL)
4095 kobj_module_path = default_path;
4096 else
4097 default_path = kobj_module_path;
4098
4099 ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
4100 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4101
4102 ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
4103 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4104
4105 /*
4106 * Move symbol tables from boot memory to ksyms_arena.
4107 */
4108 for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
4109 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
4110 kobj_export_module(mod(lp));
4111 }
4112 }
4113
4114 caddr_t
4115 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
4116 {
4117 uintptr_t va, pva;
4118 size_t alloc_pgsz = kobj_mmu_pagesize;
4119 size_t alloc_align = BO_NO_ALIGN;
4120 size_t alloc_size;
4121
4122 /*
4123 * If we are using "large" mappings for the kernel,
4124 * request aligned memory from boot using the
4125 * "large" pagesize.
4126 */
4127 if (lg_pagesize) {
4128 alloc_align = lg_pagesize;
4129 alloc_pgsz = lg_pagesize;
4130 }
4131
4132 #if defined(__sparc)
4133 /* account for redzone */
4134 if (limit)
4135 limit -= alloc_pgsz;
4136 #endif /* __sparc */
4137
4138 va = ALIGN((uintptr_t)*spp, align);
4139 pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4140 /*
4141 * Need more pages?
4142 */
4143 if (va + size > pva) {
4144 uintptr_t npva;
4145
4146 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4147 /*
4148 * Check for overlapping segments.
4149 */
4150 if (limit && limit <= *spp + alloc_size) {
4151 return ((caddr_t)0);
4152 }
4153
4154 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4155 alloc_size, alloc_align);
4156
4157 if (npva == 0) {
4158 _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4159 alloc_size);
4160 _kobj_printf(ops, " aligned %lx", alloc_align);
4161 _kobj_printf(ops, " at 0x%lx\n", pva);
4162 return (NULL);
4163 }
4164 }
4165 *spp = (caddr_t)(va + size);
4166
4167 return ((caddr_t)va);
4168 }
4169
4170 /*
4171 * Calculate the number of output hash buckets.
4172 * We use the next prime larger than n / 4,
4173 * so the average hash chain is about 4 entries.
4174 * More buckets would just be a waste of memory.
4175 */
4176 uint_t
4177 kobj_gethashsize(uint_t n)
4178 {
4179 int f;
4180 int hsize = MAX(n / 4, 2);
4181
4182 for (f = 2; f * f <= hsize; f++)
4183 if (hsize % f == 0)
4184 hsize += f = 1;
4185
4186 return (hsize);
4187 }
4188
4189 /*
4190 * Get the file size.
4191 *
4192 * Before root is mounted, files are compressed in the boot_archive ramdisk
4193 * (in the memory). kobj_fstat would return the compressed file size.
4194 * In order to get the uncompressed file size, read the file to the end and
4195 * count its size.
4196 */
4197 int
4198 kobj_get_filesize(struct _buf *file, uint64_t *size)
4199 {
4200 int err = 0;
4201 ssize_t resid;
4202 uint32_t buf;
4203
4204 if (_modrootloaded) {
4205 struct bootstat bst;
4206
4207 if (kobj_fstat(file->_fd, &bst) != 0)
4208 return (EIO);
4209 *size = bst.st_size;
4210
4211 if (file->_iscmp == CH_MAGIC_GZIP) {
4212 /*
4213 * Read the last 4 bytes of the compressed (gzip)
4214 * image to get the size of its uncompressed
4215 * version.
4216 */
4217 if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4218 (char *)(&buf), 4, (offset_t)(*size - 4),
4219 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid))
4220 != 0) {
4221 _kobj_printf(ops, "kobj_get_filesize: "
4222 "vn_rdwr() failed with error 0x%x\n", err);
4223 return (-1);
4224 }
4225
4226 *size = (uint64_t)buf;
4227 }
4228 } else {
4229
4230 #if defined(_OBP)
4231 struct bootstat bsb;
4232
4233 if (file->_iscmp) {
4234 struct comphdr *hdr = kobj_comphdr(file);
4235
4236 *size = hdr->ch_fsize;
4237 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4238 return (EIO);
4239 else
4240 *size = bsb.st_size;
4241 #else
4242 char *buf;
4243 int count;
4244 uint64_t offset = 0;
4245
4246 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4247 do {
4248 count = kobj_read_file(file, buf, MAXBSIZE, offset);
4249 if (count < 0) {
4250 kmem_free(buf, MAXBSIZE);
4251 return (EIO);
4252 }
4253 offset += count;
4254 } while (count == MAXBSIZE);
4255 kmem_free(buf, MAXBSIZE);
4256
4257 *size = offset;
4258 #endif
4259 }
4260
4261 return (0);
4262 }
4263
4264 static char *
4265 basename(char *s)
4266 {
4267 char *p, *q;
4268
4269 q = NULL;
4270 p = s;
4271 do {
4272 if (*p == '/')
4273 q = p;
4274 } while (*p++);
4275 return (q ? q + 1 : s);
4276 }
4277
4278 void
4279 kobj_stat_get(kobj_stat_t *kp)
4280 {
4281 *kp = kobj_stat;
4282 }
4283
4284 int
4285 kobj_getpagesize()
4286 {
4287 return (lg_pagesize);
4288 }
4289
4290 void
4291 kobj_textwin_alloc(struct module *mp)
4292 {
4293 ASSERT(MUTEX_HELD(&mod_lock));
4294
4295 if (mp->textwin != NULL)
4296 return;
4297
4298 /*
4299 * If the text is not contained in the heap, then it is not contained
4300 * by a writable mapping. (Specifically, it's on the nucleus page.)
4301 * We allocate a read/write mapping for this module's text to allow
4302 * the text to be patched without calling hot_patch_kernel_text()
4303 * (which is quite slow).
4304 */
4305 if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4306 uintptr_t text = (uintptr_t)mp->text;
4307 uintptr_t size = (uintptr_t)mp->text_size;
4308 uintptr_t i;
4309 caddr_t va;
4310 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4311 (text & PAGEMASK);
4312
4313 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4314
4315 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4316 hat_devload(kas.a_hat, va, PAGESIZE,
4317 hat_getpfnum(kas.a_hat, (caddr_t)i),
4318 PROT_READ | PROT_WRITE,
4319 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4320 va += PAGESIZE;
4321 }
4322
4323 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4324 } else {
4325 mp->textwin = mp->text;
4326 }
4327 }
4328
4329 void
4330 kobj_textwin_free(struct module *mp)
4331 {
4332 uintptr_t text = (uintptr_t)mp->text;
4333 uintptr_t tsize = (uintptr_t)mp->text_size;
4334 size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4335 (text & PAGEMASK));
4336
4337 mp->textwin = NULL;
4338
4339 if (mp->textwin_base == NULL)
4340 return;
4341
4342 hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4343 vmem_free(heap_arena, mp->textwin_base, size);
4344 mp->textwin_base = NULL;
4345 }
4346
4347 static char *
4348 find_libmacro(char *name)
4349 {
4350 int lmi;
4351
4352 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4353 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4354 return (libmacros[lmi].lmi_list);
4355 }
4356 return (NULL);
4357 }
4358
4359 /*
4360 * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4361 * returns path if successful, else NULL
4362 * Support multiple $MACROs expansion and the first valid path will be returned
4363 * Caller's responsibility to provide enough space in path to expand
4364 */
4365 char *
4366 expand_libmacro(char *tail, char *path, char *pathend)
4367 {
4368 char c, *p, *p1, *p2, *path2, *endp;
4369 int diff, lmi, macrolen, valid_macro, more_macro;
4370 struct _buf *file;
4371
4372 /*
4373 * check for $MACROS between nulls or slashes
4374 */
4375 p = strchr(tail, '$');
4376 if (p == NULL)
4377 return (NULL);
4378 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4379 macrolen = libmacros[lmi].lmi_macrolen;
4380 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4381 break;
4382 }
4383
4384 valid_macro = 0;
4385 if (lmi < NLIBMACROS) {
4386 /*
4387 * The following checks are used to restrict expansion of
4388 * macros to those that form a full directory/file name
4389 * and to keep the behavior same as before. If this
4390 * restriction is removed or no longer valid in the future,
4391 * the checks below can be deleted.
4392 */
4393 if ((p == tail) || (*(p - 1) == '/')) {
4394 c = *(p + macrolen + 1);
4395 if (c == '/' || c == '\0')
4396 valid_macro = 1;
4397 }
4398 }
4399
4400 if (!valid_macro) {
4401 p2 = strchr(p, '/');
4402 /*
4403 * if no more macro to expand, then just copy whatever left
4404 * and check whether it exists
4405 */
4406 if (p2 == NULL || strchr(p2, '$') == NULL) {
4407 (void) strcpy(pathend, tail);
4408 if ((file = kobj_open_path(path, 1, 1)) !=
4409 (struct _buf *)-1) {
4410 kobj_close_file(file);
4411 return (path);
4412 } else
4413 return (NULL);
4414 } else {
4415 /*
4416 * copy all chars before '/' and call expand_libmacro()
4417 * again
4418 */
4419 diff = p2 - tail;
4420 bcopy(tail, pathend, diff);
4421 pathend += diff;
4422 *(pathend) = '\0';
4423 return (expand_libmacro(p2, path, pathend));
4424 }
4425 }
4426
4427 more_macro = 0;
4428 if (c != '\0') {
4429 endp = p + macrolen + 1;
4430 if (strchr(endp, '$') != NULL)
4431 more_macro = 1;
4432 } else
4433 endp = NULL;
4434
4435 /*
4436 * copy lmi_list and split it into components.
4437 * then put the part of tail before $MACRO into path
4438 * at pathend
4439 */
4440 diff = p - tail;
4441 if (diff > 0)
4442 bcopy(tail, pathend, diff);
4443 path2 = pathend + diff;
4444 p1 = libmacros[lmi].lmi_list;
4445 while (p1 && (*p1 != '\0')) {
4446 p2 = strchr(p1, ':');
4447 if (p2) {
4448 diff = p2 - p1;
4449 bcopy(p1, path2, diff);
4450 *(path2 + diff) = '\0';
4451 } else {
4452 diff = strlen(p1);
4453 bcopy(p1, path2, diff + 1);
4454 }
4455 /* copy endp only if there isn't any more macro to expand */
4456 if (!more_macro && (endp != NULL))
4457 (void) strcat(path2, endp);
4458 file = kobj_open_path(path, 1, 1);
4459 if (file != (struct _buf *)-1) {
4460 kobj_close_file(file);
4461 /*
4462 * if more macros to expand then call expand_libmacro(),
4463 * else return path which has the whole path
4464 */
4465 if (!more_macro || (expand_libmacro(endp, path,
4466 path2 + diff) != NULL)) {
4467 return (path);
4468 }
4469 }
4470 if (p2)
4471 p1 = ++p2;
4472 else
4473 return (NULL);
4474 }
4475 return (NULL);
4476 }
4477
4478 static void
4479 tnf_add_notifyunload(kobj_notify_f *fp)
4480 {
4481 kobj_notify_list_t *entry;
4482
4483 entry = kobj_alloc(sizeof (kobj_notify_list_t), KM_WAIT);
4484 entry->kn_type = KOBJ_NOTIFY_MODUNLOADING;
4485 entry->kn_func = fp;
4486 (void) kobj_notify_add(entry);
4487 }
4488
4489 /* ARGSUSED */
4490 static void
4491 tnf_unsplice_probes(uint_t what, struct modctl *mod)
4492 {
4493 tnf_probe_control_t **p;
4494 tnf_tag_data_t **q;
4495 struct module *mp = mod->mod_mp;
4496
4497 if (!(mp->flags & KOBJ_TNF_PROBE))
4498 return;
4499
4500 for (p = &__tnf_probe_list_head; *p; )
4501 if (kobj_addrcheck(mp, (char *)*p) == 0)
4502 *p = (*p)->next;
4503 else
4504 p = &(*p)->next;
4505
4506 for (q = &__tnf_tag_list_head; *q; )
4507 if (kobj_addrcheck(mp, (char *)*q) == 0)
4508 *q = (tnf_tag_data_t *)(*q)->tag_version;
4509 else
4510 q = (tnf_tag_data_t **)&(*q)->tag_version;
4511
4512 tnf_changed_probe_list = 1;
4513 }
4514
4515 int
4516 tnf_splice_probes(int boot_load, tnf_probe_control_t *plist,
4517 tnf_tag_data_t *tlist)
4518 {
4519 int result = 0;
4520 static int add_notify = 1;
4521
4522 if (plist) {
4523 tnf_probe_control_t *pl;
4524
4525 for (pl = plist; pl->next; )
4526 pl = pl->next;
4527
4528 if (!boot_load)
4529 mutex_enter(&mod_lock);
4530 tnf_changed_probe_list = 1;
4531 pl->next = __tnf_probe_list_head;
4532 __tnf_probe_list_head = plist;
4533 if (!boot_load)
4534 mutex_exit(&mod_lock);
4535 result = 1;
4536 }
4537
4538 if (tlist) {
4539 tnf_tag_data_t *tl;
4540
4541 for (tl = tlist; tl->tag_version; )
4542 tl = (tnf_tag_data_t *)tl->tag_version;
4543
4544 if (!boot_load)
4545 mutex_enter(&mod_lock);
4546 tl->tag_version = (tnf_tag_version_t *)__tnf_tag_list_head;
4547 __tnf_tag_list_head = tlist;
4548 if (!boot_load)
4549 mutex_exit(&mod_lock);
4550 result = 1;
4551 }
4552 if (!boot_load && result && add_notify) {
4553 tnf_add_notifyunload(tnf_unsplice_probes);
4554 add_notify = 0;
4555 }
4556 return (result);
4557 }
4558
4559 char *kobj_file_buf;
4560 int kobj_file_bufsize;
4561
4562 /*
4563 * This code is for the purpose of manually recording which files
4564 * needs to go into the boot archive on any given system.
4565 *
4566 * To enable the code, set kobj_file_bufsize in /etc/system
4567 * and reboot the system, then use mdb to look at kobj_file_buf.
4568 */
4569 static void
4570 kobj_record_file(char *filename)
4571 {
4572 static char *buf;
4573 static int size = 0;
4574 int n;
4575
4576 if (kobj_file_bufsize == 0) /* don't bother */
4577 return;
4578
4579 if (kobj_file_buf == NULL) { /* allocate buffer */
4580 size = kobj_file_bufsize;
4581 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4582 }
4583
4584 n = snprintf(buf, size, "%s\n", filename);
4585 if (n > size)
4586 n = size;
4587 size -= n;
4588 buf += n;
4589 }
4590
4591 static int
4592 kobj_boot_fstat(int fd, struct bootstat *stp)
4593 {
4594 #if defined(_OBP)
4595 if (!standalone && _ioquiesced)
4596 return (-1);
4597 return (BOP_FSTAT(ops, fd, stp));
4598 #else
4599 return (BRD_FSTAT(bfs_ops, fd, stp));
4600 #endif
4601 }
4602
4603 static int
4604 kobj_boot_open(char *filename, int flags)
4605 {
4606 #if defined(_OBP)
4607
4608 /*
4609 * If io via bootops is quiesced, it means boot is no longer
4610 * available to us. We make it look as if we can't open the
4611 * named file - which is reasonably accurate.
4612 */
4613 if (!standalone && _ioquiesced)
4614 return (-1);
4615
4616 kobj_record_file(filename);
4617 return (BOP_OPEN(filename, flags));
4618 #else /* x86 */
4619 kobj_record_file(filename);
4620 return (BRD_OPEN(bfs_ops, filename, flags));
4621 #endif
4622 }
4623
4624 static int
4625 kobj_boot_close(int fd)
4626 {
4627 #if defined(_OBP)
4628 if (!standalone && _ioquiesced)
4629 return (-1);
4630
4631 return (BOP_CLOSE(fd));
4632 #else /* x86 */
4633 return (BRD_CLOSE(bfs_ops, fd));
4634 #endif
4635 }
4636
4637 /*ARGSUSED*/
4638 static int
4639 kobj_boot_seek(int fd, off_t hi, off_t lo)
4640 {
4641 #if defined(_OBP)
4642 return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4643 #else
4644 return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4645 #endif
4646 }
4647
4648 static int
4649 kobj_boot_read(int fd, caddr_t buf, size_t size)
4650 {
4651 #if defined(_OBP)
4652 return (BOP_READ(fd, buf, size));
4653 #else
4654 return (BRD_READ(bfs_ops, fd, buf, size));
4655 #endif
4656 }
4657
4658 static int
4659 kobj_boot_compinfo(int fd, struct compinfo *cb)
4660 {
4661 return (boot_compinfo(fd, cb));
4662 }
4663
4664 /*
4665 * Check if the file is compressed (for now we handle only gzip).
4666 * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4667 */
4668 static int
4669 kobj_is_compressed(intptr_t fd)
4670 {
4671 struct vnode *vp = (struct vnode *)fd;
4672 ssize_t resid;
4673 uint16_t magic_buf;
4674 int err = 0;
4675
4676 if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4677 sizeof (magic_buf), (offset_t)(0),
4678 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
4679
4680 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4681 "error code 0x%x\n", err);
4682 return (0);
4683 }
4684
4685 if (magic_buf == CH_MAGIC_GZIP)
4686 return (CH_MAGIC_GZIP);
4687
4688 return (0);
4689 }