Print this page
9059 Simplify SMAP relocations with krtld
Portions contributed by: John Levon <john.levon@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/krtld/kobj.c
+++ new/usr/src/uts/common/krtld/kobj.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25 /*
26 26 * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
27 27 * All rights reserved. Use is subject to license terms.
28 28 * Copyright 2020 Joyent, Inc.
29 29 */
30 30
31 31 /*
32 32 * Kernel's linker/loader
33 33 */
34 34
35 35 #include <sys/types.h>
36 36 #include <sys/param.h>
37 37 #include <sys/sysmacros.h>
38 38 #include <sys/systm.h>
39 39 #include <sys/user.h>
40 40 #include <sys/kmem.h>
41 41 #include <sys/reboot.h>
42 42 #include <sys/bootconf.h>
43 43 #include <sys/debug.h>
44 44 #include <sys/uio.h>
45 45 #include <sys/file.h>
46 46 #include <sys/vnode.h>
47 47 #include <sys/user.h>
48 48 #include <sys/mman.h>
49 49 #include <vm/as.h>
50 50 #include <vm/seg_kp.h>
51 51 #include <vm/seg_kmem.h>
52 52 #include <sys/elf.h>
53 53 #include <sys/elf_notes.h>
54 54 #include <sys/vmsystm.h>
55 55 #include <sys/kdi.h>
56 56 #include <sys/atomic.h>
57 57 #include <sys/kmdb.h>
58 58
59 59 #include <sys/link.h>
60 60 #include <sys/kobj.h>
61 61 #include <sys/ksyms.h>
62 62 #include <sys/disp.h>
63 63 #include <sys/modctl.h>
64 64 #include <sys/varargs.h>
65 65 #include <sys/kstat.h>
66 66 #include <sys/kobj_impl.h>
67 67 #include <sys/fs/decomp.h>
68 68 #include <sys/callb.h>
69 69 #include <sys/cmn_err.h>
70 70 #include <sys/tnf_probe.h>
71 71 #include <sys/zmod.h>
72 72
73 73 #include <krtld/reloc.h>
74 74 #include <krtld/kobj_kdi.h>
75 75 #include <sys/sha1.h>
76 76 #include <sys/crypto/elfsign.h>
77 77
78 78 #if !defined(_OBP)
79 79 #include <sys/bootvfs.h>
80 80 #endif
81 81
82 82 /*
83 83 * do_symbols() error codes
84 84 */
85 85 #define DOSYM_UNDEF -1 /* undefined symbol */
86 86 #define DOSYM_UNSAFE -2 /* MT-unsafe driver symbol */
87 87
88 88 #if !defined(_OBP)
89 89 static void synthetic_bootaux(char *, val_t *);
90 90 #endif
91 91
92 92 static struct module *load_exec(val_t *, char *);
93 93 static void load_linker(val_t *);
94 94 static struct modctl *add_primary(const char *filename, int);
95 95 static int bind_primary(val_t *, int);
96 96 static int load_primary(struct module *, int);
97 97 static int load_kmdb(val_t *);
98 98 static int get_progbits(struct module *, struct _buf *);
99 99 static int get_syms(struct module *, struct _buf *);
100 100 static int get_ctf(struct module *, struct _buf *);
101 101 static void get_signature(struct module *, struct _buf *);
102 102 static int do_common(struct module *);
103 103 static void add_dependent(struct module *, struct module *);
104 104 static int do_dependents(struct modctl *, char *, size_t);
105 105 static int do_symbols(struct module *, Elf64_Addr);
106 106 static void module_assign(struct modctl *, struct module *);
107 107 static void free_module_data(struct module *);
108 108 static char *depends_on(struct module *);
109 109 static char *getmodpath(const char *);
110 110 static char *basename(char *);
111 111 static void attr_val(val_t *);
112 112 static char *find_libmacro(char *);
113 113 static char *expand_libmacro(char *, char *, char *);
114 114 static int read_bootflags(void);
115 115 static int kobj_comp_setup(struct _buf *, struct compinfo *);
116 116 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
117 117 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
118 118 static int kobj_boot_open(char *, int);
119 119 static int kobj_boot_close(int);
120 120 static int kobj_boot_seek(int, off_t, off_t);
121 121 static int kobj_boot_read(int, caddr_t, size_t);
122 122 static int kobj_boot_fstat(int, struct bootstat *);
123 123 static int kobj_boot_compinfo(int, struct compinfo *);
124 124
125 125 static Sym *lookup_one(struct module *, const char *);
126 126 static void sym_insert(struct module *, char *, symid_t);
127 127 static Sym *sym_lookup(struct module *, Sym *);
128 128
129 129 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
130 130 static void kobjopen_free(struct kobjopen_tctl *ltp);
131 131 static void kobjopen_thread(struct kobjopen_tctl *ltp);
132 132 static int kobj_is_compressed(intptr_t);
133 133
134 134 extern int kcopy(const void *, void *, size_t);
135 135 extern int elf_mach_ok(Ehdr *);
136 136 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
137 137
138 138 #if !defined(_OBP)
139 139 extern int kobj_boot_mountroot(void);
140 140 #endif
141 141
142 142 static void tnf_unsplice_probes(uint_t, struct modctl *);
143 143 extern tnf_probe_control_t *__tnf_probe_list_head;
144 144 extern tnf_tag_data_t *__tnf_tag_list_head;
145 145
146 146 extern int modrootloaded;
147 147 extern int swaploaded;
148 148 extern int bop_io_quiesced;
149 149 extern int last_module_id;
150 150
151 151 extern char stubs_base[];
152 152 extern char stubs_end[];
153 153
154 154 #ifdef KOBJ_DEBUG
155 155 /*
156 156 * Values that can be or'd in to kobj_debug and their effects:
157 157 *
158 158 * D_DEBUG - misc. debugging information.
159 159 * D_SYMBOLS - list symbols and their values as they are entered
160 160 * into the hash table
161 161 * D_RELOCATIONS - display relocation processing information
162 162 * D_LOADING - display information about each module as it
163 163 * is loaded.
164 164 */
165 165 int kobj_debug = 0;
166 166
167 167 #define KOBJ_MARK(s) if (kobj_debug & D_DEBUG) \
168 168 (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
169 169 #else
170 170 #define KOBJ_MARK(s) /* discard */
171 171 #endif
172 172
173 173 #define MODPATH_PROPNAME "module-path"
174 174
175 175 #ifdef MODDIR_SUFFIX
176 176 static char slash_moddir_suffix_slash[] = MODDIR_SUFFIX "/";
177 177 #else
178 178 #define slash_moddir_suffix_slash ""
179 179 #endif
180 180
181 181 #define _moddebug get_weakish_int(&moddebug)
182 182 #define _modrootloaded get_weakish_int(&modrootloaded)
183 183 #define _swaploaded get_weakish_int(&swaploaded)
184 184 #define _ioquiesced get_weakish_int(&bop_io_quiesced)
185 185
186 186 #define mod(X) (struct module *)((X)->modl_modp->mod_mp)
187 187
188 188 void *romp; /* rom vector (opaque to us) */
189 189 struct bootops *ops; /* bootops vector */
190 190 void *dbvec; /* debug vector */
191 191
192 192 /*
193 193 * kobjopen thread control structure
194 194 */
195 195 struct kobjopen_tctl {
196 196 ksema_t sema;
197 197 char *name; /* name of file */
198 198 struct vnode *vp; /* vnode return from vn_open() */
199 199 int Errno; /* error return from vnopen */
200 200 };
201 201
202 202 /*
203 203 * Structure for defining dynamically expandable library macros
204 204 */
205 205
206 206 struct lib_macro_info {
207 207 char *lmi_list; /* ptr to list of possible choices */
208 208 char *lmi_macroname; /* pointer to macro name */
209 209 ushort_t lmi_ba_index; /* index into bootaux vector */
210 210 ushort_t lmi_macrolen; /* macro length */
211 211 } libmacros[] = {
212 212 { NULL, "CPU", BA_CPU, 0 },
213 213 { NULL, "MMU", BA_MMU, 0 }
214 214 };
215 215
216 216 #define NLIBMACROS sizeof (libmacros) / sizeof (struct lib_macro_info)
217 217
218 218 char *boot_cpu_compatible_list; /* make $CPU available */
219 219
220 220 char *kobj_module_path; /* module search path */
221 221 vmem_t *text_arena; /* module text arena */
222 222 static vmem_t *data_arena; /* module data & bss arena */
223 223 static vmem_t *ctf_arena; /* CTF debug data arena */
224 224 static struct modctl *kobj_modules = NULL; /* modules loaded */
225 225 int kobj_mmu_pagesize; /* system pagesize */
226 226 static int lg_pagesize; /* "large" pagesize */
227 227 static int kobj_last_module_id = 0; /* id assignment */
228 228 static kmutex_t kobj_lock; /* protects mach memory list */
229 229
230 230 /*
231 231 * The following functions have been implemented by the kernel.
232 232 * However, many 3rd party drivers provide their own implementations
233 233 * of these functions. When such drivers are loaded, messages
234 234 * indicating that these symbols have been multiply defined will be
235 235 * emitted to the console. To avoid alarming customers for no good
236 236 * reason, we simply suppress such warnings for the following set of
237 237 * functions.
238 238 */
239 239 static char *suppress_sym_list[] =
240 240 {
241 241 "strstr",
242 242 "strncat",
243 243 "strlcat",
244 244 "strlcpy",
245 245 "strspn",
246 246 "memcpy",
247 247 "memset",
248 248 "memmove",
249 249 "memcmp",
250 250 "memchr",
251 251 "__udivdi3",
252 252 "__divdi3",
253 253 "__umoddi3",
254 254 "__moddi3",
255 255 NULL /* This entry must exist */
256 256 };
257 257
258 258 /* indexed by KOBJ_NOTIFY_* */
259 259 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
260 260
261 261 /*
262 262 * TNF probe management globals
263 263 */
264 264 tnf_probe_control_t *__tnf_probe_list_head = NULL;
265 265 tnf_tag_data_t *__tnf_tag_list_head = NULL;
266 266 int tnf_changed_probe_list = 0;
267 267
268 268 /*
269 269 * Prefix for statically defined tracing (SDT) DTrace probes.
270 270 */
271 271 const char *sdt_prefix = "__dtrace_probe_";
272 272
273 273 /*
274 274 * Beginning and end of the kernel's dynamic text/data segments.
275 275 */
276 276 static caddr_t _text;
277 277 static caddr_t _etext;
278 278 static caddr_t _data;
279 279
280 280 /*
281 281 * The sparc linker doesn't create a memory location
282 282 * for a variable named _edata, so _edata can only be
283 283 * referred to, not modified. krtld needs a static
284 284 * variable to modify it - within krtld, of course -
285 285 * outside of krtld, e_data is used in all kernels.
286 286 */
287 287 #if defined(__sparc)
288 288 static caddr_t _edata;
289 289 #else
290 290 extern caddr_t _edata;
291 291 #endif
292 292
293 293 Addr dynseg = 0; /* load address of "dynamic" segment */
294 294 size_t dynsize; /* "dynamic" segment size */
295 295
296 296
297 297 int standalone = 1; /* an unwholey kernel? */
298 298 int use_iflush; /* iflush after relocations */
299 299
300 300 /*
301 301 * _kobj_printf() and _vkobj_printf()
302 302 *
303 303 * Common printf function pointer. Can handle only one conversion
304 304 * specification in the format string. Some of the functions invoked
305 305 * through this function pointer cannot handle more that one conversion
306 306 * specification in the format string.
307 307 */
308 308 void (*_kobj_printf)(void *, const char *, ...) __KPRINTFLIKE(2);
309 309 void (*_vkobj_printf)(void *, const char *, va_list) __KVPRINTFLIKE(2);
310 310
311 311 /*
312 312 * Standalone function pointers for use within krtld.
313 313 * Many platforms implement optimized platmod versions of
314 314 * utilities such as bcopy and any such are not yet available
315 315 * until the kernel is more completely stitched together.
316 316 * See kobj_impl.h
317 317 */
318 318 void (*kobj_bcopy)(const void *, void *, size_t);
319 319 void (*kobj_bzero)(void *, size_t);
320 320 size_t (*kobj_strlcat)(char *, const char *, size_t);
321 321
322 322 static kobj_stat_t kobj_stat;
323 323
324 324 #define MINALIGN 8 /* at least a double-word */
325 325
326 326 int
327 327 get_weakish_int(int *ip)
328 328 {
329 329 if (standalone)
330 330 return (0);
331 331 return (ip == NULL ? 0 : *ip);
332 332 }
333 333
334 334 static void *
335 335 get_weakish_pointer(void **ptrp)
336 336 {
337 337 if (standalone)
338 338 return (0);
339 339 return (ptrp == NULL ? 0 : *ptrp);
340 340 }
341 341
342 342 /*
343 343 * XXX fix dependencies on "kernel"; this should work
344 344 * for other standalone binaries as well.
345 345 *
346 346 * XXX Fix hashing code to use one pointer to
347 347 * hash entries.
348 348 * |----------|
349 349 * | nbuckets |
350 350 * |----------|
351 351 * | nchains |
352 352 * |----------|
353 353 * | bucket[] |
354 354 * |----------|
355 355 * | chain[] |
356 356 * |----------|
357 357 */
358 358
359 359 /*
360 360 * Load, bind and relocate all modules that
361 361 * form the primary kernel. At this point, our
362 362 * externals have not been relocated.
363 363 */
364 364 void
365 365 kobj_init(
366 366 void *romvec,
367 367 void *dvec,
368 368 struct bootops *bootvec,
369 369 val_t *bootaux)
370 370 {
371 371 struct module *mp;
372 372 struct modctl *modp;
373 373 Addr entry;
374 374 char filename[MAXPATHLEN];
375 375
376 376 /*
377 377 * Save these to pass on to
378 378 * the booted standalone.
379 379 */
380 380 romp = romvec;
381 381 dbvec = dvec;
382 382
383 383 ops = bootvec;
384 384 kobj_setup_standalone_vectors();
385 385
386 386 KOBJ_MARK("Entered kobj_init()");
387 387
388 388 (void) BOP_GETPROP(ops, "whoami", filename);
389 389
390 390 /*
391 391 * We don't support standalone debuggers anymore. The use of kadb
392 392 * will interfere with the later use of kmdb. Let the user mend
393 393 * their ways now. Users will reach this message if they still
394 394 * have the kadb binary on their system (perhaps they used an old
395 395 * bfu, or maybe they intentionally copied it there) and have
396 396 * specified its use in a way that eluded our checking in the boot
397 397 * program.
398 398 */
399 399 if (dvec != NULL) {
400 400 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
401 401 "kadb are no longer supported\n\n");
402 402 goto fail;
403 403 }
404 404
405 405 #if defined(_OBP)
406 406 /*
407 407 * OBP allows us to read both the ramdisk and
408 408 * the underlying root fs when root is a disk.
409 409 * This can lower incidences of unbootable systems
410 410 * when the archive is out-of-date with the /etc
411 411 * state files.
412 412 */
413 413 if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
414 414 _kobj_printf(ops, "can't mount boot fs\n");
415 415 goto fail;
416 416 }
417 417 #else
418 418 {
419 419 /* on x86, we always boot with a ramdisk */
420 420 (void) kobj_boot_mountroot();
421 421
422 422 /*
423 423 * Now that the ramdisk is mounted, finish boot property
424 424 * initialization.
425 425 */
426 426 read_bootenvrc();
427 427 }
428 428
429 429 #if !defined(_UNIX_KRTLD)
430 430 /*
431 431 * 'unix' is linked together with 'krtld' into one executable and
432 432 * the early boot code does -not- hand us any of the dynamic metadata
433 433 * about the executable. In particular, it does not read in, map or
434 434 * otherwise look at the program headers. We fake all that up now.
435 435 *
436 436 * We do this early as DTrace static probes and tnf probes both call
437 437 * undefined references. We have to process those relocations before
438 438 * calling any of them.
439 439 *
440 440 * OBP tells kobj_start() where the ELF image is in memory, so it
441 441 * synthesized bootaux before kobj_init() was called
442 442 */
443 443 if (bootaux[BA_PHDR].ba_ptr == NULL)
444 444 synthetic_bootaux(filename, bootaux);
445 445
446 446 #endif /* !_UNIX_KRTLD */
447 447 #endif /* _OBP */
448 448
449 449 /*
450 450 * Save the interesting attribute-values
451 451 * (scanned by kobj_boot).
452 452 */
453 453 attr_val(bootaux);
454 454
455 455 /*
456 456 * Set the module search path.
457 457 */
458 458 kobj_module_path = getmodpath(filename);
459 459
460 460 boot_cpu_compatible_list = find_libmacro("CPU");
461 461
462 462 /*
463 463 * These two modules have actually been
464 464 * loaded by boot, but we finish the job
465 465 * by introducing them into the world of
466 466 * loadable modules.
467 467 */
468 468
469 469 mp = load_exec(bootaux, filename);
470 470 load_linker(bootaux);
471 471
472 472 /*
473 473 * Load all the primary dependent modules.
474 474 */
475 475 if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
476 476 goto fail;
477 477
478 478 /*
479 479 * Glue it together.
480 480 */
481 481 if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
482 482 goto fail;
483 483
484 484 entry = bootaux[BA_ENTRY].ba_val;
485 485
486 486 /*
487 487 * Get the boot flags
488 488 */
489 489 bootflags(ops);
490 490
491 491 if (boothowto & RB_VERBOSE)
492 492 kobj_lm_dump(KOBJ_LM_PRIMARY);
493 493
494 494 kobj_kdi_init();
495 495
496 496 if (boothowto & RB_KMDB) {
497 497 if (load_kmdb(bootaux) < 0)
498 498 goto fail;
499 499 }
500 500
501 501 /*
502 502 * Post setup.
503 503 */
504 504 s_text = _text;
505 505 e_text = _etext;
506 506 s_data = _data;
507 507 e_data = _edata;
508 508
509 509 kobj_sync_instruction_memory(s_text, e_text - s_text);
510 510
511 511 #ifdef KOBJ_DEBUG
512 512 if (kobj_debug & D_DEBUG)
513 513 _kobj_printf(ops,
514 514 "krtld: transferring control to: 0x%lx\n", entry);
515 515 #endif
516 516
517 517 /*
518 518 * Make sure the mod system knows about the modules already loaded.
519 519 */
520 520 last_module_id = kobj_last_module_id;
521 521 bcopy(kobj_modules, &modules, sizeof (modules));
522 522 modp = &modules;
523 523 do {
524 524 if (modp->mod_next == kobj_modules)
525 525 modp->mod_next = &modules;
526 526 if (modp->mod_prev == kobj_modules)
527 527 modp->mod_prev = &modules;
528 528 } while ((modp = modp->mod_next) != &modules);
529 529
530 530 standalone = 0;
531 531
532 532 #ifdef KOBJ_DEBUG
533 533 if (kobj_debug & D_DEBUG)
534 534 _kobj_printf(ops,
535 535 "krtld: really transferring control to: 0x%lx\n", entry);
536 536 #endif
537 537
538 538 /* restore printf/bcopy/bzero vectors before returning */
539 539 kobj_restore_vectors();
540 540
541 541 #if defined(_DBOOT)
542 542 /*
543 543 * krtld was called from a dboot ELF section, the embedded
544 544 * dboot code contains the real entry via bootaux
545 545 */
546 546 exitto((caddr_t)entry);
547 547 #else
548 548 /*
549 549 * krtld was directly called from startup
550 550 */
551 551 return;
552 552 #endif
553 553
554 554 fail:
555 555
556 556 _kobj_printf(ops, "krtld: error during initial load/link phase\n");
557 557
558 558 #if !defined(_UNIX_KRTLD)
559 559 _kobj_printf(ops, "\n");
560 560 _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
561 561 " for:\n");
562 562 _kobj_printf(ops, " %s\n", filename);
563 563 _kobj_printf(ops, "in the boot archive. Please verify that this"
564 564 " file\n");
565 565 _kobj_printf(ops, "matches what is found in the boot archive.\n");
566 566 _kobj_printf(ops, "You may need to boot using the Solaris failsafe to"
567 567 " fix this.\n");
568 568 bop_panic("Unable to boot");
569 569 #endif
570 570 }
571 571
572 572 #if !defined(_UNIX_KRTLD) && !defined(_OBP)
573 573 /*
574 574 * Synthesize additional metadata that describes the executable if
575 575 * krtld's caller didn't do it.
576 576 *
577 577 * (When the dynamic executable has an interpreter, the boot program
578 578 * does all this for us. Where we don't have an interpreter, (or a
579 579 * even a boot program, perhaps) we have to do this for ourselves.)
580 580 */
581 581 static void
582 582 synthetic_bootaux(char *filename, val_t *bootaux)
583 583 {
584 584 Ehdr ehdr;
585 585 caddr_t phdrbase;
586 586 struct _buf *file;
587 587 int i, n;
588 588
589 589 /*
590 590 * Elf header
591 591 */
592 592 KOBJ_MARK("synthetic_bootaux()");
593 593 KOBJ_MARK(filename);
594 594 file = kobj_open_file(filename);
595 595 if (file == (struct _buf *)-1) {
596 596 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
597 597 return;
598 598 }
599 599 KOBJ_MARK("reading program headers");
600 600 if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
601 601 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
602 602 filename);
603 603 return;
604 604 }
605 605
606 606 /*
607 607 * Program headers
608 608 */
609 609 bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
610 610 bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
611 611 n = ehdr.e_phentsize * ehdr.e_phnum;
612 612
613 613 phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
614 614
615 615 if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
616 616 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
617 617 filename);
618 618 return;
619 619 }
620 620 bootaux[BA_PHDR].ba_ptr = phdrbase;
621 621 kobj_close_file(file);
622 622 KOBJ_MARK("closed file");
623 623
624 624 /*
625 625 * Find the dynamic section address
626 626 */
627 627 for (i = 0; i < ehdr.e_phnum; i++) {
628 628 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
629 629
630 630 if (phdr->p_type == PT_DYNAMIC) {
631 631 bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
632 632 break;
633 633 }
634 634 }
635 635 KOBJ_MARK("synthetic_bootaux() done");
636 636 }
637 637 #endif /* !_UNIX_KRTLD && !_OBP */
638 638
639 639 /*
640 640 * Set up any global information derived
641 641 * from attribute/values in the boot or
642 642 * aux vector.
643 643 */
644 644 static void
645 645 attr_val(val_t *bootaux)
646 646 {
647 647 Phdr *phdr;
648 648 int phnum, phsize;
649 649 int i;
650 650
651 651 KOBJ_MARK("attr_val()");
652 652 kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
653 653 lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
654 654 use_iflush = bootaux[BA_IFLUSH].ba_val;
655 655
656 656 phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
657 657 phnum = bootaux[BA_PHNUM].ba_val;
658 658 phsize = bootaux[BA_PHENT].ba_val;
659 659 for (i = 0; i < phnum; i++) {
660 660 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
661 661
662 662 if (phdr->p_type != PT_LOAD) {
663 663 continue;
664 664 }
665 665 /*
666 666 * Bounds of the various segments.
667 667 */
668 668 if (!(phdr->p_flags & PF_X)) {
669 669 #if defined(_RELSEG)
670 670 /*
671 671 * sparc kernel puts the dynamic info
672 672 * into a separate segment, which is
673 673 * free'd in bop_fini()
674 674 */
675 675 ASSERT(phdr->p_vaddr != 0);
676 676 dynseg = phdr->p_vaddr;
677 677 dynsize = phdr->p_memsz;
678 678 #else
679 679 ASSERT(phdr->p_vaddr == 0);
680 680 #endif
681 681 } else {
682 682 if (phdr->p_flags & PF_W) {
683 683 _data = (caddr_t)phdr->p_vaddr;
684 684 _edata = _data + phdr->p_memsz;
685 685 } else {
686 686 _text = (caddr_t)phdr->p_vaddr;
687 687 _etext = _text + phdr->p_memsz;
688 688 }
689 689 }
690 690 }
691 691
692 692 /* To do the kobj_alloc, _edata needs to be set. */
693 693 for (i = 0; i < NLIBMACROS; i++) {
694 694 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
695 695 libmacros[i].lmi_list = kobj_alloc(
696 696 strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
697 697 1, KM_WAIT);
698 698 (void) strcpy(libmacros[i].lmi_list,
699 699 bootaux[libmacros[i].lmi_ba_index].ba_ptr);
700 700 }
701 701 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
702 702 }
703 703 }
704 704
705 705 /*
706 706 * Set up the booted executable.
707 707 */
708 708 static struct module *
709 709 load_exec(val_t *bootaux, char *filename)
710 710 {
711 711 struct modctl *cp;
712 712 struct module *mp;
713 713 Dyn *dyn;
714 714 Sym *sp;
715 715 int i, lsize, osize, nsize, allocsize;
716 716 char *libname, *tmp;
717 717 char path[MAXPATHLEN];
718 718
719 719 #ifdef KOBJ_DEBUG
720 720 if (kobj_debug & D_DEBUG)
721 721 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
722 722 #endif
723 723
724 724 KOBJ_MARK("add_primary");
725 725 cp = add_primary(filename, KOBJ_LM_PRIMARY);
726 726
727 727 KOBJ_MARK("struct module");
728 728 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
729 729 cp->mod_mp = mp;
730 730
731 731 /*
732 732 * We don't have the following information
733 733 * since this module is an executable and not
734 734 * a relocatable .o.
735 735 */
736 736 mp->symtbl_section = 0;
737 737 mp->shdrs = NULL;
738 738 mp->strhdr = NULL;
739 739
740 740 /*
741 741 * Since this module is the only exception,
742 742 * we cons up some section headers.
743 743 */
744 744 KOBJ_MARK("symhdr");
745 745 mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
746 746
747 747 KOBJ_MARK("strhdr");
748 748 mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
749 749
750 750 mp->symhdr->sh_type = SHT_SYMTAB;
751 751 mp->strhdr->sh_type = SHT_STRTAB;
752 752 /*
753 753 * Scan the dynamic structure.
754 754 */
755 755 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
756 756 dyn->d_tag != DT_NULL; dyn++) {
757 757 switch (dyn->d_tag) {
758 758 case DT_SYMTAB:
759 759 mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
760 760 mp->symhdr->sh_addr = dyn->d_un.d_ptr;
761 761 break;
762 762 case DT_HASH:
763 763 mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
764 764 mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
765 765 break;
766 766 case DT_STRTAB:
767 767 mp->strings = (char *)dyn->d_un.d_ptr;
768 768 mp->strhdr->sh_addr = dyn->d_un.d_ptr;
769 769 break;
770 770 case DT_STRSZ:
771 771 mp->strhdr->sh_size = dyn->d_un.d_val;
772 772 break;
773 773 case DT_SYMENT:
774 774 mp->symhdr->sh_entsize = dyn->d_un.d_val;
775 775 break;
776 776 }
777 777 }
778 778
779 779 /*
780 780 * Collapse any DT_NEEDED entries into one string.
781 781 */
782 782 nsize = osize = 0;
783 783 allocsize = MAXPATHLEN;
784 784
785 785 KOBJ_MARK("depends_on");
786 786 mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
787 787
788 788 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
789 789 dyn->d_tag != DT_NULL; dyn++)
790 790 if (dyn->d_tag == DT_NEEDED) {
791 791 char *_lib;
792 792
793 793 libname = mp->strings + dyn->d_un.d_val;
794 794 if (strchr(libname, '$') != NULL) {
795 795 if ((_lib = expand_libmacro(libname,
796 796 path, path)) != NULL)
797 797 libname = _lib;
798 798 else
799 799 _kobj_printf(ops, "krtld: "
800 800 "load_exec: fail to "
801 801 "expand %s\n", libname);
802 802 }
803 803 lsize = strlen(libname);
804 804 nsize += lsize;
805 805 if (nsize + 1 > allocsize) {
806 806 KOBJ_MARK("grow depends_on");
807 807 tmp = kobj_alloc(allocsize + MAXPATHLEN,
808 808 KM_WAIT);
809 809 bcopy(mp->depends_on, tmp, osize);
810 810 kobj_free(mp->depends_on, allocsize);
811 811 mp->depends_on = tmp;
812 812 allocsize += MAXPATHLEN;
813 813 }
814 814 bcopy(libname, mp->depends_on + osize, lsize);
815 815 *(mp->depends_on + nsize) = ' '; /* separate */
816 816 nsize++;
817 817 osize = nsize;
818 818 }
819 819 if (nsize) {
820 820 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
821 821 /*
822 822 * alloc with exact size and copy whatever it got over
823 823 */
824 824 KOBJ_MARK("realloc depends_on");
825 825 tmp = kobj_alloc(nsize, KM_WAIT);
826 826 bcopy(mp->depends_on, tmp, nsize);
827 827 kobj_free(mp->depends_on, allocsize);
828 828 mp->depends_on = tmp;
829 829 } else {
830 830 kobj_free(mp->depends_on, allocsize);
831 831 mp->depends_on = NULL;
832 832 }
833 833
834 834 mp->flags = KOBJ_EXEC|KOBJ_PRIM; /* NOT a relocatable .o */
835 835 mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
836 836 /*
837 837 * We allocate our own table since we don't
838 838 * hash undefined references.
839 839 */
840 840 KOBJ_MARK("chains");
841 841 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
842 842 KOBJ_MARK("buckets");
843 843 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
844 844
845 845 mp->text = _text;
846 846 mp->data = _data;
847 847
848 848 mp->text_size = _etext - _text;
849 849 mp->data_size = _edata - _data;
850 850
851 851 cp->mod_text = mp->text;
852 852 cp->mod_text_size = mp->text_size;
853 853
854 854 mp->filename = cp->mod_filename;
855 855
856 856 #ifdef KOBJ_DEBUG
857 857 if (kobj_debug & D_LOADING) {
858 858 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
859 859 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
860 860 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
861 861 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
862 862 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
863 863 }
864 864 #endif /* KOBJ_DEBUG */
865 865
866 866 /*
867 867 * Insert symbols into the hash table.
868 868 */
869 869 for (i = 0; i < mp->nsyms; i++) {
870 870 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
871 871
872 872 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
873 873 continue;
874 874 #if defined(__sparc)
875 875 /*
876 876 * Register symbols are ignored in the kernel
877 877 */
878 878 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER)
879 879 continue;
880 880 #endif /* __sparc */
881 881
882 882 sym_insert(mp, mp->strings + sp->st_name, i);
883 883 }
884 884
885 885 KOBJ_MARK("load_exec done");
886 886 return (mp);
887 887 }
888 888
889 889 /*
890 890 * Set up the linker module (if it's compiled in, LDNAME is NULL)
891 891 */
892 892 static void
893 893 load_linker(val_t *bootaux)
894 894 {
895 895 struct module *kmp = (struct module *)kobj_modules->mod_mp;
896 896 struct module *mp;
897 897 struct modctl *cp;
898 898 int i;
899 899 Shdr *shp;
900 900 Sym *sp;
901 901 int shsize;
902 902 char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
903 903
904 904 /*
905 905 * On some architectures, krtld is compiled into the kernel.
906 906 */
907 907 if (dlname == NULL)
908 908 return;
909 909
910 910 cp = add_primary(dlname, KOBJ_LM_PRIMARY);
911 911
912 912 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
913 913
914 914 cp->mod_mp = mp;
915 915 mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
916 916 shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
917 917 mp->shdrs = kobj_alloc(shsize, KM_WAIT);
918 918 bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
919 919
920 920 for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
921 921 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
922 922
923 923 if (shp->sh_flags & SHF_ALLOC) {
924 924 if (shp->sh_flags & SHF_WRITE) {
925 925 if (mp->data == NULL)
926 926 mp->data = (char *)shp->sh_addr;
927 927 } else if (mp->text == NULL) {
928 928 mp->text = (char *)shp->sh_addr;
929 929 }
930 930 }
931 931 if (shp->sh_type == SHT_SYMTAB) {
932 932 mp->symtbl_section = i;
933 933 mp->symhdr = shp;
934 934 mp->symspace = mp->symtbl = (char *)shp->sh_addr;
935 935 }
936 936 }
937 937 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
938 938 mp->flags = KOBJ_INTERP|KOBJ_PRIM;
939 939 mp->strhdr = (Shdr *)
940 940 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
941 941 mp->strings = (char *)mp->strhdr->sh_addr;
942 942 mp->hashsize = kobj_gethashsize(mp->nsyms);
943 943
944 944 mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
945 945 (mp->hashsize + mp->nsyms) * sizeof (symid_t);
946 946
947 947 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
948 948 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
949 949
950 950 mp->bss = bootaux[BA_BSS].ba_val;
951 951 mp->bss_align = 0; /* pre-aligned during allocation */
952 952 mp->bss_size = (uintptr_t)_edata - mp->bss;
953 953 mp->text_size = _etext - mp->text;
954 954 mp->data_size = _edata - mp->data;
955 955 mp->filename = cp->mod_filename;
956 956 cp->mod_text = mp->text;
957 957 cp->mod_text_size = mp->text_size;
958 958
959 959 /*
960 960 * Now that we've figured out where the linker is,
961 961 * set the limits for the booted object.
962 962 */
963 963 kmp->text_size = (size_t)(mp->text - kmp->text);
964 964 kmp->data_size = (size_t)(mp->data - kmp->data);
965 965 kobj_modules->mod_text_size = kmp->text_size;
966 966
967 967 #ifdef KOBJ_DEBUG
968 968 if (kobj_debug & D_LOADING) {
969 969 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
970 970 _kobj_printf(ops, "\ttext:0x%p", mp->text);
971 971 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
972 972 _kobj_printf(ops, "\tdata:0x%p", mp->data);
973 973 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
974 974 }
975 975 #endif /* KOBJ_DEBUG */
976 976
977 977 /*
978 978 * Insert the symbols into the hash table.
979 979 */
980 980 for (i = 0; i < mp->nsyms; i++) {
981 981 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
982 982
983 983 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
984 984 continue;
985 985 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
986 986 if (sp->st_shndx == SHN_COMMON)
987 987 sp->st_shndx = SHN_ABS;
988 988 }
989 989 sym_insert(mp, mp->strings + sp->st_name, i);
990 990 }
991 991
992 992 }
993 993
994 994 static kobj_notify_list_t **
995 995 kobj_notify_lookup(uint_t type)
996 996 {
997 997 ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
998 998 sizeof (kobj_notify_list_t *));
999 999
1000 1000 return (&kobj_notifiers[type]);
1001 1001 }
1002 1002
1003 1003 int
1004 1004 kobj_notify_add(kobj_notify_list_t *knp)
1005 1005 {
1006 1006 kobj_notify_list_t **knl;
1007 1007
1008 1008 knl = kobj_notify_lookup(knp->kn_type);
1009 1009
1010 1010 knp->kn_next = NULL;
1011 1011 knp->kn_prev = NULL;
1012 1012
1013 1013 mutex_enter(&kobj_lock);
1014 1014
1015 1015 if (*knl != NULL) {
1016 1016 (*knl)->kn_prev = knp;
1017 1017 knp->kn_next = *knl;
1018 1018 }
1019 1019 (*knl) = knp;
1020 1020
1021 1021 mutex_exit(&kobj_lock);
1022 1022 return (0);
1023 1023 }
1024 1024
1025 1025 int
1026 1026 kobj_notify_remove(kobj_notify_list_t *knp)
1027 1027 {
1028 1028 kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
1029 1029 kobj_notify_list_t *tknp;
1030 1030
1031 1031 mutex_enter(&kobj_lock);
1032 1032
1033 1033 /* LINTED */
1034 1034 if (tknp = knp->kn_next)
1035 1035 tknp->kn_prev = knp->kn_prev;
1036 1036
1037 1037 /* LINTED */
1038 1038 if (tknp = knp->kn_prev)
1039 1039 tknp->kn_next = knp->kn_next;
1040 1040 else
1041 1041 *knl = knp->kn_next;
1042 1042
1043 1043 mutex_exit(&kobj_lock);
1044 1044
1045 1045 return (0);
1046 1046 }
1047 1047
1048 1048 /*
1049 1049 * Notify all interested callbacks of a specified change in module state.
1050 1050 */
1051 1051 static void
1052 1052 kobj_notify(int type, struct modctl *modp)
1053 1053 {
1054 1054 kobj_notify_list_t *knp;
1055 1055
1056 1056 if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1057 1057 return;
1058 1058
1059 1059 mutex_enter(&kobj_lock);
1060 1060
1061 1061 for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1062 1062 knp->kn_func(type, modp);
1063 1063
1064 1064 /*
1065 1065 * KDI notification must be last (it has to allow for work done by the
1066 1066 * other notification callbacks), so we call it manually.
1067 1067 */
1068 1068 kobj_kdi_mod_notify(type, modp);
1069 1069
1070 1070 mutex_exit(&kobj_lock);
1071 1071 }
1072 1072
1073 1073 /*
1074 1074 * Create the module path.
1075 1075 */
1076 1076 static char *
1077 1077 getmodpath(const char *filename)
1078 1078 {
1079 1079 char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1080 1080
1081 1081 /*
1082 1082 * Platform code gets first crack, then add
1083 1083 * the default components
1084 1084 */
1085 1085 mach_modpath(path, filename);
1086 1086 if (*path != '\0')
1087 1087 (void) strcat(path, " ");
1088 1088 return (strcat(path, MOD_DEFPATH));
1089 1089 }
1090 1090
1091 1091 static struct modctl *
1092 1092 add_primary(const char *filename, int lmid)
1093 1093 {
1094 1094 struct modctl *cp;
1095 1095
1096 1096 cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1097 1097
1098 1098 cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1099 1099
1100 1100 /*
1101 1101 * For symbol lookup, we assemble our own
1102 1102 * modctl list of the primary modules.
1103 1103 */
1104 1104
1105 1105 (void) strcpy(cp->mod_filename, filename);
1106 1106 cp->mod_modname = basename(cp->mod_filename);
1107 1107
1108 1108 /* set values for modinfo assuming that the load will work */
1109 1109 cp->mod_prim = 1;
1110 1110 cp->mod_loaded = 1;
1111 1111 cp->mod_installed = 1;
1112 1112 cp->mod_loadcnt = 1;
1113 1113 cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1114 1114
1115 1115 cp->mod_id = kobj_last_module_id++;
1116 1116
1117 1117 /*
1118 1118 * Link the module in. We'll pass this info on
1119 1119 * to the mod squad later.
1120 1120 */
1121 1121 if (kobj_modules == NULL) {
1122 1122 kobj_modules = cp;
1123 1123 cp->mod_prev = cp->mod_next = cp;
1124 1124 } else {
1125 1125 cp->mod_prev = kobj_modules->mod_prev;
1126 1126 cp->mod_next = kobj_modules;
1127 1127 kobj_modules->mod_prev->mod_next = cp;
1128 1128 kobj_modules->mod_prev = cp;
1129 1129 }
1130 1130
1131 1131 kobj_lm_append(lmid, cp);
1132 1132
1133 1133 return (cp);
1134 1134 }
1135 1135
1136 1136 static int
1137 1137 bind_primary(val_t *bootaux, int lmid)
1138 1138 {
1139 1139 struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1140 1140 struct modctl_list *lp;
1141 1141 struct module *mp;
1142 1142
1143 1143 /*
1144 1144 * Do common symbols.
1145 1145 */
1146 1146 for (lp = linkmap; lp; lp = lp->modl_next) {
1147 1147 mp = mod(lp);
1148 1148
1149 1149 /*
1150 1150 * Don't do common section relocations for modules that
1151 1151 * don't need it.
1152 1152 */
1153 1153 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1154 1154 continue;
1155 1155
1156 1156 if (do_common(mp) < 0)
1157 1157 return (-1);
1158 1158 }
1159 1159
1160 1160 /*
1161 1161 * Resolve symbols.
1162 1162 */
1163 1163 for (lp = linkmap; lp; lp = lp->modl_next) {
1164 1164 mp = mod(lp);
1165 1165
1166 1166 if (do_symbols(mp, 0) < 0)
1167 1167 return (-1);
1168 1168 }
1169 1169
1170 1170 /*
1171 1171 * Do relocations.
1172 1172 */
1173 1173 for (lp = linkmap; lp; lp = lp->modl_next) {
1174 1174 mp = mod(lp);
1175 1175
1176 1176 if (mp->flags & KOBJ_EXEC) {
1177 1177 Dyn *dyn;
1178 1178 Word relasz = 0, relaent = 0;
1179 1179 char *rela = NULL;
1180 1180
1181 1181 for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1182 1182 dyn->d_tag != DT_NULL; dyn++) {
1183 1183 switch (dyn->d_tag) {
1184 1184 case DT_RELASZ:
1185 1185 case DT_RELSZ:
1186 1186 relasz = dyn->d_un.d_val;
1187 1187 break;
1188 1188 case DT_RELAENT:
1189 1189 case DT_RELENT:
1190 1190 relaent = dyn->d_un.d_val;
1191 1191 break;
1192 1192 case DT_RELA:
1193 1193 rela = (char *)dyn->d_un.d_ptr;
1194 1194 break;
1195 1195 case DT_REL:
1196 1196 rela = (char *)dyn->d_un.d_ptr;
1197 1197 break;
1198 1198 }
1199 1199 }
1200 1200 if (relasz == 0 ||
1201 1201 relaent == 0 || rela == NULL) {
1202 1202 _kobj_printf(ops, "krtld: bind_primary(): "
1203 1203 "no relocation information found for "
1204 1204 "module %s\n", mp->filename);
1205 1205 return (-1);
1206 1206 }
1207 1207 #ifdef KOBJ_DEBUG
1208 1208 if (kobj_debug & D_RELOCATIONS)
1209 1209 _kobj_printf(ops, "krtld: relocating: file=%s "
1210 1210 "KOBJ_EXEC\n", mp->filename);
1211 1211 #endif
1212 1212 if (do_relocate(mp, rela, relasz/relaent, relaent,
1213 1213 (Addr)mp->text) < 0)
1214 1214 return (-1);
1215 1215 } else {
1216 1216 if (do_relocations(mp) < 0)
1217 1217 return (-1);
1218 1218 }
1219 1219
1220 1220 kobj_sync_instruction_memory(mp->text, mp->text_size);
1221 1221 }
1222 1222
1223 1223 for (lp = linkmap; lp; lp = lp->modl_next) {
1224 1224 mp = mod(lp);
1225 1225
1226 1226 /*
1227 1227 * We need to re-read the full symbol table for the boot file,
1228 1228 * since we couldn't use the full one before. We also need to
1229 1229 * load the CTF sections of both the boot file and the
1230 1230 * interpreter (us).
1231 1231 */
1232 1232 if (mp->flags & KOBJ_EXEC) {
1233 1233 struct _buf *file;
1234 1234 int n;
1235 1235
1236 1236 file = kobj_open_file(mp->filename);
1237 1237 if (file == (struct _buf *)-1)
1238 1238 return (-1);
1239 1239 if (kobj_read_file(file, (char *)&mp->hdr,
1240 1240 sizeof (mp->hdr), 0) < 0)
1241 1241 return (-1);
1242 1242 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1243 1243 mp->shdrs = kobj_alloc(n, KM_WAIT);
1244 1244 if (kobj_read_file(file, mp->shdrs, n,
1245 1245 mp->hdr.e_shoff) < 0)
1246 1246 return (-1);
1247 1247 if (get_syms(mp, file) < 0)
1248 1248 return (-1);
1249 1249 if (get_ctf(mp, file) < 0)
1250 1250 return (-1);
1251 1251 kobj_close_file(file);
1252 1252 mp->flags |= KOBJ_RELOCATED;
1253 1253
1254 1254 } else if (mp->flags & KOBJ_INTERP) {
1255 1255 struct _buf *file;
1256 1256
1257 1257 /*
1258 1258 * The interpreter path fragment in mp->filename
1259 1259 * will already have the module directory suffix
1260 1260 * in it (if appropriate).
1261 1261 */
1262 1262 file = kobj_open_path(mp->filename, 1, 0);
1263 1263 if (file == (struct _buf *)-1)
1264 1264 return (-1);
1265 1265 if (get_ctf(mp, file) < 0)
1266 1266 return (-1);
1267 1267 kobj_close_file(file);
1268 1268 mp->flags |= KOBJ_RELOCATED;
1269 1269 }
1270 1270 }
1271 1271
1272 1272 return (0);
1273 1273 }
1274 1274
1275 1275 static struct modctl *
1276 1276 mod_already_loaded(char *modname)
1277 1277 {
1278 1278 struct modctl *mctl = kobj_modules;
1279 1279
1280 1280 do {
1281 1281 if (strcmp(modname, mctl->mod_filename) == 0)
1282 1282 return (mctl);
1283 1283 mctl = mctl->mod_next;
1284 1284
1285 1285 } while (mctl != kobj_modules);
1286 1286
1287 1287 return (NULL);
1288 1288 }
1289 1289
1290 1290 /*
1291 1291 * Load all the primary dependent modules.
1292 1292 */
1293 1293 static int
1294 1294 load_primary(struct module *mp, int lmid)
1295 1295 {
1296 1296 struct modctl *cp;
1297 1297 struct module *dmp;
1298 1298 char *p, *q;
1299 1299 char modname[MODMAXNAMELEN];
1300 1300
1301 1301 if ((p = mp->depends_on) == NULL)
1302 1302 return (0);
1303 1303
1304 1304 /* CONSTANTCONDITION */
1305 1305 while (1) {
1306 1306 /*
1307 1307 * Skip space.
1308 1308 */
1309 1309 while (*p && (*p == ' ' || *p == '\t'))
1310 1310 p++;
1311 1311 /*
1312 1312 * Get module name.
1313 1313 */
1314 1314 q = modname;
1315 1315 while (*p && *p != ' ' && *p != '\t')
1316 1316 *q++ = *p++;
1317 1317
1318 1318 if (q == modname)
1319 1319 break;
1320 1320
1321 1321 *q = '\0';
1322 1322 /*
1323 1323 * Check for dup dependencies.
1324 1324 */
1325 1325 if (strcmp(modname, "dtracestubs") == 0 ||
1326 1326 mod_already_loaded(modname) != NULL)
1327 1327 continue;
1328 1328
1329 1329 cp = add_primary(modname, lmid);
1330 1330 cp->mod_busy = 1;
1331 1331 /*
1332 1332 * Load it.
1333 1333 */
1334 1334 (void) kobj_load_module(cp, 1);
1335 1335 cp->mod_busy = 0;
1336 1336
1337 1337 if ((dmp = cp->mod_mp) == NULL) {
1338 1338 cp->mod_loaded = 0;
1339 1339 cp->mod_installed = 0;
1340 1340 cp->mod_loadcnt = 0;
1341 1341 return (-1);
1342 1342 }
1343 1343
1344 1344 add_dependent(mp, dmp);
1345 1345 dmp->flags |= KOBJ_PRIM;
1346 1346
1347 1347 /*
1348 1348 * Recurse.
1349 1349 */
1350 1350 if (load_primary(dmp, lmid) == -1) {
1351 1351 cp->mod_loaded = 0;
1352 1352 cp->mod_installed = 0;
1353 1353 cp->mod_loadcnt = 0;
1354 1354 return (-1);
1355 1355 }
1356 1356 }
1357 1357 return (0);
1358 1358 }
1359 1359
1360 1360 static int
1361 1361 console_is_usb_serial(void)
1362 1362 {
1363 1363 char *console;
1364 1364 int len, ret;
1365 1365
1366 1366 if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1367 1367 return (0);
1368 1368
1369 1369 console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1370 1370 (void) BOP_GETPROP(ops, "console", console);
1371 1371 ret = (strcmp(console, "usb-serial") == 0);
1372 1372 kobj_free(console, len);
1373 1373
1374 1374 return (ret);
1375 1375 }
1376 1376
1377 1377 static int
1378 1378 load_kmdb(val_t *bootaux)
1379 1379 {
1380 1380 struct modctl *mctl;
1381 1381 struct module *mp;
1382 1382 Sym *sym;
1383 1383
1384 1384 if (console_is_usb_serial()) {
1385 1385 _kobj_printf(ops, "kmdb not loaded "
1386 1386 "(unsupported on usb serial console)\n");
1387 1387 return (0);
1388 1388 }
1389 1389
1390 1390 _kobj_printf(ops, "Loading kmdb...\n");
1391 1391
1392 1392 if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1393 1393 return (-1);
1394 1394
1395 1395 mctl->mod_busy = 1;
1396 1396 (void) kobj_load_module(mctl, 1);
1397 1397 mctl->mod_busy = 0;
1398 1398
1399 1399 if ((mp = mctl->mod_mp) == NULL)
1400 1400 return (-1);
1401 1401
1402 1402 mp->flags |= KOBJ_PRIM;
1403 1403
1404 1404 if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1405 1405 return (-1);
1406 1406
1407 1407 if (boothowto & RB_VERBOSE)
1408 1408 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1409 1409
1410 1410 if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1411 1411 return (-1);
1412 1412
1413 1413 if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1414 1414 return (-1);
1415 1415
1416 1416 #ifdef KOBJ_DEBUG
1417 1417 if (kobj_debug & D_DEBUG) {
1418 1418 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1419 1419 sym->st_value);
1420 1420 _kobj_printf(ops, "\tops 0x%p\n", ops);
1421 1421 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1422 1422 }
1423 1423 #endif
1424 1424
1425 1425 if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1426 1426 (const char **)kobj_kmdb_argv) < 0)
1427 1427 return (-1);
1428 1428
1429 1429 return (0);
1430 1430 }
1431 1431
1432 1432 /*
1433 1433 * Return a string listing module dependencies.
1434 1434 */
1435 1435 static char *
1436 1436 depends_on(struct module *mp)
1437 1437 {
1438 1438 Sym *sp;
1439 1439 char *depstr, *q;
1440 1440
1441 1441 /*
1442 1442 * The module doesn't have a depends_on value, so let's try it the
1443 1443 * old-fashioned way - via "_depends_on"
1444 1444 */
1445 1445 if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1446 1446 return (NULL);
1447 1447
1448 1448 q = (char *)sp->st_value;
1449 1449
1450 1450 #ifdef KOBJ_DEBUG
1451 1451 /*
1452 1452 * _depends_on is a deprecated interface, so we warn about its use
1453 1453 * irrespective of subsequent processing errors. How else are we going
1454 1454 * to be able to deco this interface completely?
1455 1455 * Changes initially limited to DEBUG because third-party modules
1456 1456 * should be flagged to developers before general use base.
1457 1457 */
1458 1458 _kobj_printf(ops,
1459 1459 "Warning: %s uses deprecated _depends_on interface.\n",
1460 1460 mp->filename);
1461 1461 _kobj_printf(ops, "Please notify module developer or vendor.\n");
1462 1462 #endif
1463 1463
1464 1464 /*
1465 1465 * Idiot checks. Make sure it's
1466 1466 * in-bounds and NULL terminated.
1467 1467 */
1468 1468 if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1469 1469 _kobj_printf(ops, "Error processing dependency for %s\n",
1470 1470 mp->filename);
1471 1471 return (NULL);
1472 1472 }
1473 1473
1474 1474 depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1475 1475 (void) strcpy(depstr, q);
1476 1476
1477 1477 return (depstr);
1478 1478 }
1479 1479
1480 1480 void
1481 1481 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1482 1482 {
1483 1483 struct module *mp;
1484 1484 mp = (struct module *)xmp;
1485 1485
1486 1486 modinfo->mi_base = mp->text;
1487 1487 modinfo->mi_size = mp->text_size + mp->data_size;
1488 1488 }
1489 1489
1490 1490 /*
1491 1491 * kobj_export_ksyms() performs the following services:
1492 1492 *
1493 1493 * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1494 1494 * (2) Removes unneeded symbols to save space.
1495 1495 * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1496 1496 * (4) Makes the symbol table visible to /dev/ksyms.
1497 1497 */
1498 1498 static void
1499 1499 kobj_export_ksyms(struct module *mp)
1500 1500 {
1501 1501 Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1502 1502 Sym *sp, *osp;
1503 1503 char *name;
1504 1504 size_t namelen;
1505 1505 struct module *omp;
1506 1506 uint_t nsyms;
1507 1507 size_t symsize = mp->symhdr->sh_entsize;
1508 1508 size_t locals = 1;
1509 1509 size_t strsize;
1510 1510
1511 1511 /*
1512 1512 * Make a copy of the original module structure.
1513 1513 */
1514 1514 omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1515 1515 bcopy(mp, omp, sizeof (struct module));
1516 1516
1517 1517 /*
1518 1518 * Compute the sizes of the new symbol table sections.
1519 1519 */
1520 1520 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1521 1521 if (osp->st_value == 0)
1522 1522 continue;
1523 1523 if (sym_lookup(omp, osp) == NULL)
1524 1524 continue;
1525 1525 name = omp->strings + osp->st_name;
1526 1526 namelen = strlen(name);
1527 1527 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1528 1528 locals++;
1529 1529 nsyms++;
1530 1530 strsize += namelen + 1;
1531 1531 }
1532 1532
1533 1533 mp->nsyms = nsyms;
1534 1534 mp->hashsize = kobj_gethashsize(mp->nsyms);
1535 1535
1536 1536 /*
1537 1537 * ksyms_lock must be held as writer during any operation that
1538 1538 * modifies ksyms_arena, including allocation from same, and
1539 1539 * must not be dropped until the arena is vmem_walk()able.
1540 1540 */
1541 1541 rw_enter(&ksyms_lock, RW_WRITER);
1542 1542
1543 1543 /*
1544 1544 * Allocate space for the new section headers (symtab and strtab),
1545 1545 * symbol table, buckets, chains, and strings.
1546 1546 */
1547 1547 mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1548 1548 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1549 1549
1550 1550 if (mp->flags & KOBJ_NOKSYMS) {
1551 1551 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1552 1552 } else {
1553 1553 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1554 1554 VM_BESTFIT | VM_SLEEP);
1555 1555 }
1556 1556 bzero(mp->symspace, mp->symsize);
1557 1557
1558 1558 /*
1559 1559 * Divvy up symspace.
1560 1560 */
1561 1561 mp->shdrs = mp->symspace;
1562 1562 mp->symhdr = (Shdr *)mp->shdrs;
1563 1563 mp->strhdr = (Shdr *)(mp->symhdr + 1);
1564 1564 mp->symtbl = (char *)(mp->strhdr + 1);
1565 1565 mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1566 1566 mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1567 1567 mp->strings = (char *)(mp->chains + nsyms);
1568 1568
1569 1569 /*
1570 1570 * Fill in the new section headers (symtab and strtab).
1571 1571 */
1572 1572 mp->hdr.e_shnum = 2;
1573 1573 mp->symtbl_section = 0;
1574 1574
1575 1575 mp->symhdr->sh_type = SHT_SYMTAB;
1576 1576 mp->symhdr->sh_addr = (Addr)mp->symtbl;
1577 1577 mp->symhdr->sh_size = nsyms * symsize;
1578 1578 mp->symhdr->sh_link = 1;
1579 1579 mp->symhdr->sh_info = locals;
1580 1580 mp->symhdr->sh_addralign = sizeof (Addr);
1581 1581 mp->symhdr->sh_entsize = symsize;
1582 1582
1583 1583 mp->strhdr->sh_type = SHT_STRTAB;
1584 1584 mp->strhdr->sh_addr = (Addr)mp->strings;
1585 1585 mp->strhdr->sh_size = strsize;
1586 1586 mp->strhdr->sh_addralign = 1;
1587 1587
1588 1588 /*
1589 1589 * Construct the new symbol table.
1590 1590 */
1591 1591 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1592 1592 if (osp->st_value == 0)
1593 1593 continue;
1594 1594 if (sym_lookup(omp, osp) == NULL)
1595 1595 continue;
1596 1596 name = omp->strings + osp->st_name;
1597 1597 namelen = strlen(name);
1598 1598 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1599 1599 bcopy(osp, sp, symsize);
1600 1600 bcopy(name, mp->strings + strsize, namelen);
1601 1601 sp->st_name = strsize;
1602 1602 sym_insert(mp, name, nsyms);
1603 1603 nsyms++;
1604 1604 strsize += namelen + 1;
1605 1605 }
1606 1606
1607 1607 rw_exit(&ksyms_lock);
1608 1608
1609 1609 /*
1610 1610 * Free the old section headers -- we'll never need them again.
1611 1611 */
1612 1612 if (!(mp->flags & KOBJ_PRIM)) {
1613 1613 uint_t shn;
1614 1614 Shdr *shp;
1615 1615
1616 1616 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1617 1617 shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1618 1618 switch (shp->sh_type) {
1619 1619 case SHT_RELA:
1620 1620 case SHT_REL:
1621 1621 if (shp->sh_addr != 0) {
1622 1622 kobj_free((void *)shp->sh_addr,
1623 1623 shp->sh_size);
1624 1624 }
1625 1625 break;
1626 1626 }
1627 1627 }
1628 1628 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1629 1629 }
1630 1630 /*
1631 1631 * Discard the old symbol table and our copy of the module strucure.
1632 1632 */
1633 1633 if (!(mp->flags & KOBJ_PRIM))
1634 1634 kobj_free(omp->symspace, omp->symsize);
1635 1635 kobj_free(omp, sizeof (struct module));
1636 1636 }
1637 1637
1638 1638 static void
1639 1639 kobj_export_ctf(struct module *mp)
1640 1640 {
1641 1641 char *data = mp->ctfdata;
1642 1642 size_t size = mp->ctfsize;
1643 1643
1644 1644 if (data != NULL) {
1645 1645 if (_moddebug & MODDEBUG_NOCTF) {
1646 1646 mp->ctfdata = NULL;
1647 1647 mp->ctfsize = 0;
1648 1648 } else {
1649 1649 mp->ctfdata = vmem_alloc(ctf_arena, size,
1650 1650 VM_BESTFIT | VM_SLEEP);
1651 1651 bcopy(data, mp->ctfdata, size);
1652 1652 }
1653 1653
1654 1654 if (!(mp->flags & KOBJ_PRIM))
1655 1655 kobj_free(data, size);
1656 1656 }
1657 1657 }
1658 1658
1659 1659 void
1660 1660 kobj_export_module(struct module *mp)
1661 1661 {
1662 1662 kobj_export_ksyms(mp);
1663 1663 kobj_export_ctf(mp);
1664 1664
1665 1665 mp->flags |= KOBJ_EXPORTED;
1666 1666 }
1667 1667
1668 1668 static int
1669 1669 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1670 1670 {
1671 1671 char *path = NULL, *depstr = NULL;
1672 1672 int allocsize = 0, osize = 0, nsize = 0;
1673 1673 char *libname, *tmp;
1674 1674 int lsize;
1675 1675 Dyn *dynp;
1676 1676
1677 1677 for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1678 1678 switch (dynp->d_tag) {
1679 1679 case DT_NEEDED:
1680 1680 /*
1681 1681 * Read the DT_NEEDED entries, expanding the macros they
1682 1682 * contain (if any), and concatenating them into a
1683 1683 * single space-separated dependency list.
1684 1684 */
1685 1685 libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1686 1686
1687 1687 if (strchr(libname, '$') != NULL) {
1688 1688 char *_lib;
1689 1689
1690 1690 if (path == NULL)
1691 1691 path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1692 1692 if ((_lib = expand_libmacro(libname, path,
1693 1693 path)) != NULL)
1694 1694 libname = _lib;
1695 1695 else {
1696 1696 _kobj_printf(ops, "krtld: "
1697 1697 "process_dynamic: failed to expand "
1698 1698 "%s\n", libname);
1699 1699 }
1700 1700 }
1701 1701
1702 1702 lsize = strlen(libname);
1703 1703 nsize += lsize;
1704 1704 if (nsize + 1 > allocsize) {
1705 1705 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1706 1706 KM_WAIT);
1707 1707 if (depstr != NULL) {
1708 1708 bcopy(depstr, tmp, osize);
1709 1709 kobj_free(depstr, allocsize);
1710 1710 }
1711 1711 depstr = tmp;
1712 1712 allocsize += MAXPATHLEN;
1713 1713 }
1714 1714 bcopy(libname, depstr + osize, lsize);
1715 1715 *(depstr + nsize) = ' '; /* separator */
1716 1716 nsize++;
1717 1717 osize = nsize;
1718 1718 break;
1719 1719
1720 1720 case DT_FLAGS_1:
1721 1721 if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1722 1722 mp->flags |= KOBJ_IGNMULDEF;
1723 1723 if (dynp->d_un.d_val & DF_1_NOKSYMS)
1724 1724 mp->flags |= KOBJ_NOKSYMS;
1725 1725
1726 1726 break;
1727 1727 }
1728 1728 }
1729 1729
1730 1730 /*
1731 1731 * finish up the depends string (if any)
1732 1732 */
1733 1733 if (depstr != NULL) {
1734 1734 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1735 1735 if (path != NULL)
1736 1736 kobj_free(path, MAXPATHLEN);
1737 1737
1738 1738 tmp = kobj_alloc(nsize, KM_WAIT);
1739 1739 bcopy(depstr, tmp, nsize);
1740 1740 kobj_free(depstr, allocsize);
1741 1741 depstr = tmp;
1742 1742
1743 1743 mp->depends_on = depstr;
1744 1744 }
1745 1745
1746 1746 return (0);
1747 1747 }
1748 1748
1749 1749 static int
1750 1750 do_dynamic(struct module *mp, struct _buf *file)
1751 1751 {
1752 1752 Shdr *dshp, *dstrp, *shp;
1753 1753 char *dyndata, *dstrdata;
1754 1754 int dshn, shn, rc;
1755 1755
1756 1756 /* find and validate the dynamic section (if any) */
1757 1757
1758 1758 for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1759 1759 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1760 1760 switch (shp->sh_type) {
1761 1761 case SHT_DYNAMIC:
1762 1762 if (dshp != NULL) {
1763 1763 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1764 1764 mp->filename);
1765 1765 _kobj_printf(ops,
1766 1766 "multiple dynamic sections\n");
1767 1767 return (-1);
1768 1768 } else {
1769 1769 dshp = shp;
1770 1770 dshn = shn;
1771 1771 }
1772 1772 break;
1773 1773 }
1774 1774 }
1775 1775
1776 1776 if (dshp == NULL)
1777 1777 return (0);
1778 1778
1779 1779 if (dshp->sh_link > mp->hdr.e_shnum) {
1780 1780 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1781 1781 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1782 1782 return (-1);
1783 1783 }
1784 1784 dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1785 1785
1786 1786 if (dstrp->sh_type != SHT_STRTAB) {
1787 1787 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1788 1788 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1789 1789 dshn);
1790 1790 return (-1);
1791 1791 }
1792 1792
1793 1793 /* read it from disk */
1794 1794
1795 1795 dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1796 1796 if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1797 1797 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1798 1798 _kobj_printf(ops, "error reading section %d\n", dshn);
1799 1799
1800 1800 kobj_free(dyndata, dshp->sh_size);
1801 1801 return (-1);
1802 1802 }
1803 1803
1804 1804 dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1805 1805 if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1806 1806 dstrp->sh_offset) < 0) {
1807 1807 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1808 1808 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1809 1809
1810 1810 kobj_free(dyndata, dshp->sh_size);
1811 1811 kobj_free(dstrdata, dstrp->sh_size);
1812 1812 return (-1);
1813 1813 }
1814 1814
1815 1815 /* pull the interesting pieces out */
1816 1816
1817 1817 rc = process_dynamic(mp, dyndata, dstrdata);
1818 1818
1819 1819 kobj_free(dyndata, dshp->sh_size);
1820 1820 kobj_free(dstrdata, dstrp->sh_size);
1821 1821
1822 1822 return (rc);
1823 1823 }
1824 1824
1825 1825 void
1826 1826 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1827 1827 {
1828 1828 if (!standalone) {
1829 1829 if (mp->ctfdata != NULL) {
1830 1830 if (vmem_contains(ctf_arena, mp->ctfdata,
1831 1831 mp->ctfsize)) {
1832 1832 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1833 1833 } else {
1834 1834 kobj_free(mp->ctfdata, mp->ctfsize);
1835 1835 }
1836 1836 }
1837 1837 }
1838 1838
1839 1839 /*
1840 1840 * The order is very important here. We need to make sure that
1841 1841 * consumers, at any given instant, see a consistent state. We'd
1842 1842 * rather they see no CTF data than the address of one buffer and the
1843 1843 * size of another.
1844 1844 */
1845 1845 mp->ctfdata = NULL;
1846 1846 membar_producer();
1847 1847 mp->ctfsize = size;
1848 1848 mp->ctfdata = data;
1849 1849 membar_producer();
1850 1850 }
1851 1851
1852 1852 int
1853 1853 kobj_load_module(struct modctl *modp, int use_path)
1854 1854 {
1855 1855 char *filename = modp->mod_filename;
1856 1856 char *modname = modp->mod_modname;
1857 1857 int i;
1858 1858 int n;
1859 1859 struct _buf *file;
1860 1860 struct module *mp = NULL;
1861 1861 #ifdef MODDIR_SUFFIX
1862 1862 int no_suffixdir_drv = 0;
1863 1863 #endif
1864 1864
1865 1865 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1866 1866
1867 1867 /*
1868 1868 * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1869 1869 * kmdb contains a bunch of symbols with well-known names, symbols
1870 1870 * which will mask the real versions, thus causing no end of trouble
1871 1871 * for mdb.
1872 1872 */
1873 1873 if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1874 1874 mp->flags |= KOBJ_NOKSYMS;
1875 1875
1876 1876 file = kobj_open_path(filename, use_path, 1);
1877 1877 if (file == (struct _buf *)-1) {
1878 1878 #ifdef MODDIR_SUFFIX
1879 1879 file = kobj_open_path(filename, use_path, 0);
1880 1880 #endif
1881 1881 if (file == (struct _buf *)-1) {
1882 1882 kobj_free(mp, sizeof (*mp));
1883 1883 goto bad;
1884 1884 }
1885 1885 #ifdef MODDIR_SUFFIX
1886 1886 /*
1887 1887 * There is no driver module in the ISA specific (suffix)
1888 1888 * subdirectory but there is a module in the parent directory.
1889 1889 */
1890 1890 if (strncmp(filename, "drv/", 4) == 0) {
1891 1891 no_suffixdir_drv = 1;
1892 1892 }
1893 1893 #endif
1894 1894 }
1895 1895
1896 1896 mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1897 1897 (void) strcpy(mp->filename, file->_name);
1898 1898
1899 1899 if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1900 1900 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1901 1901 modname);
1902 1902 kobj_free(mp->filename, strlen(file->_name) + 1);
1903 1903 kobj_free(mp, sizeof (*mp));
1904 1904 goto bad;
1905 1905 }
1906 1906 for (i = 0; i < SELFMAG; i++) {
1907 1907 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1908 1908 if (_moddebug & MODDEBUG_ERRMSG)
1909 1909 _kobj_printf(ops, "%s not an elf module\n",
1910 1910 modname);
1911 1911 kobj_free(mp->filename, strlen(file->_name) + 1);
1912 1912 kobj_free(mp, sizeof (*mp));
1913 1913 goto bad;
1914 1914 }
1915 1915 }
1916 1916 /*
1917 1917 * It's ELF, but is it our ISA? Interpreting the header
1918 1918 * from a file for a byte-swapped ISA could cause a huge
1919 1919 * and unsatisfiable value to be passed to kobj_alloc below
1920 1920 * and therefore hang booting.
1921 1921 */
1922 1922 if (!elf_mach_ok(&mp->hdr)) {
1923 1923 if (_moddebug & MODDEBUG_ERRMSG)
1924 1924 _kobj_printf(ops, "%s not an elf module for this ISA\n",
1925 1925 modname);
1926 1926 kobj_free(mp->filename, strlen(file->_name) + 1);
1927 1927 kobj_free(mp, sizeof (*mp));
1928 1928 #ifdef MODDIR_SUFFIX
1929 1929 /*
1930 1930 * The driver mod is not in the ISA specific subdirectory
1931 1931 * and the module in the parent directory is not our ISA.
1932 1932 * If it is our ISA, for now we will silently succeed.
1933 1933 */
1934 1934 if (no_suffixdir_drv == 1) {
1935 1935 cmn_err(CE_CONT, "?NOTICE: %s: 64-bit driver module"
1936 1936 " not found\n", modname);
1937 1937 }
1938 1938 #endif
1939 1939 goto bad;
1940 1940 }
1941 1941
1942 1942 /*
1943 1943 * All modules, save for unix, should be relocatable (as opposed to
1944 1944 * dynamic). Dynamic modules come with PLTs and GOTs, which can't
1945 1945 * currently be processed by krtld.
1946 1946 */
1947 1947 if (mp->hdr.e_type != ET_REL) {
1948 1948 if (_moddebug & MODDEBUG_ERRMSG)
1949 1949 _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1950 1950 "module\n", modname);
1951 1951 kobj_free(mp->filename, strlen(file->_name) + 1);
1952 1952 kobj_free(mp, sizeof (*mp));
1953 1953 goto bad;
1954 1954 }
1955 1955
1956 1956 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1957 1957 mp->shdrs = kobj_alloc(n, KM_WAIT);
1958 1958
1959 1959 if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1960 1960 _kobj_printf(ops, "kobj_load_module: %s error reading "
1961 1961 "section headers\n", modname);
1962 1962 kobj_free(mp->shdrs, n);
1963 1963 kobj_free(mp->filename, strlen(file->_name) + 1);
1964 1964 kobj_free(mp, sizeof (*mp));
1965 1965 goto bad;
1966 1966 }
1967 1967
1968 1968 kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1969 1969 module_assign(modp, mp);
1970 1970
1971 1971 /* read in sections */
1972 1972 if (get_progbits(mp, file) < 0) {
1973 1973 _kobj_printf(ops, "%s error reading sections\n", modname);
1974 1974 goto bad;
1975 1975 }
1976 1976
1977 1977 if (do_dynamic(mp, file) < 0) {
1978 1978 _kobj_printf(ops, "%s error reading dynamic section\n",
1979 1979 modname);
1980 1980 goto bad;
1981 1981 }
1982 1982
1983 1983 modp->mod_text = mp->text;
1984 1984 modp->mod_text_size = mp->text_size;
1985 1985
1986 1986 /* read in symbols; adjust values for each section's real address */
1987 1987 if (get_syms(mp, file) < 0) {
1988 1988 _kobj_printf(ops, "%s error reading symbols\n",
1989 1989 modname);
1990 1990 goto bad;
1991 1991 }
1992 1992
1993 1993 /*
1994 1994 * If we didn't dependency information from the dynamic section, look
1995 1995 * for it the old-fashioned way.
1996 1996 */
1997 1997 if (mp->depends_on == NULL)
1998 1998 mp->depends_on = depends_on(mp);
1999 1999
2000 2000 if (get_ctf(mp, file) < 0) {
2001 2001 _kobj_printf(ops, "%s debug information will not "
2002 2002 "be available\n", modname);
2003 2003 }
2004 2004
2005 2005 /* primary kernel modules do not have a signature section */
2006 2006 if (!(mp->flags & KOBJ_PRIM))
2007 2007 get_signature(mp, file);
2008 2008
2009 2009 #ifdef KOBJ_DEBUG
2010 2010 if (kobj_debug & D_LOADING) {
2011 2011 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
2012 2012 _kobj_printf(ops, "\ttext:0x%p", mp->text);
2013 2013 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
2014 2014 _kobj_printf(ops, "\tdata:0x%p", mp->data);
2015 2015 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
2016 2016 }
2017 2017 #endif /* KOBJ_DEBUG */
2018 2018
2019 2019 /*
2020 2020 * For primary kernel modules, we defer
2021 2021 * symbol resolution and relocation until
2022 2022 * all primary objects have been loaded.
2023 2023 */
2024 2024 if (!standalone) {
2025 2025 int ddrval, dcrval;
2026 2026 char *dependent_modname;
2027 2027 /* load all dependents */
2028 2028 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
2029 2029 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
2030 2030
2031 2031 /*
2032 2032 * resolve undefined and common symbols,
2033 2033 * also allocates common space
2034 2034 */
2035 2035 if ((dcrval = do_common(mp)) < 0) {
2036 2036 switch (dcrval) {
2037 2037 case DOSYM_UNSAFE:
2038 2038 _kobj_printf(ops, "WARNING: mod_load: "
2039 2039 "MT-unsafe module '%s' rejected\n",
2040 2040 modname);
2041 2041 break;
2042 2042 case DOSYM_UNDEF:
2043 2043 _kobj_printf(ops, "WARNING: mod_load: "
2044 2044 "cannot load module '%s'\n",
2045 2045 modname);
2046 2046 if (ddrval == -1) {
2047 2047 _kobj_printf(ops, "WARNING: %s: ",
2048 2048 modname);
2049 2049 _kobj_printf(ops,
2050 2050 "unable to resolve dependency, "
2051 2051 "module '%s' not found\n",
2052 2052 dependent_modname);
2053 2053 }
2054 2054 break;
2055 2055 }
2056 2056 }
2057 2057 kobj_free(dependent_modname, MODMAXNAMELEN);
2058 2058 if (dcrval < 0)
2059 2059 goto bad;
2060 2060
2061 2061 /* process relocation tables */
2062 2062 if (do_relocations(mp) < 0) {
2063 2063 _kobj_printf(ops, "%s error doing relocations\n",
2064 2064 modname);
2065 2065 goto bad;
2066 2066 }
2067 2067
2068 2068 if (mp->destination) {
2069 2069 off_t off = (uintptr_t)mp->destination & PAGEOFFSET;
2070 2070 caddr_t base = (caddr_t)mp->destination - off;
2071 2071 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2072 2072
2073 2073 hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2074 2074 vmem_free(heap_arena, base, size);
2075 2075 }
2076 2076
2077 2077 /* sync_instruction_memory */
2078 2078 kobj_sync_instruction_memory(mp->text, mp->text_size);
2079 2079 kobj_export_module(mp);
2080 2080 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2081 2081 }
2082 2082 kobj_close_file(file);
2083 2083 return (0);
2084 2084 bad:
2085 2085 if (file != (struct _buf *)-1)
2086 2086 kobj_close_file(file);
2087 2087 if (modp->mod_mp != NULL)
2088 2088 free_module_data(modp->mod_mp);
2089 2089
2090 2090 module_assign(modp, NULL);
2091 2091 return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2092 2092 }
2093 2093
2094 2094 int
2095 2095 kobj_load_primary_module(struct modctl *modp)
2096 2096 {
2097 2097 struct modctl *dep;
2098 2098 struct module *mp;
2099 2099
2100 2100 if (kobj_load_module(modp, 0) != 0)
2101 2101 return (-1);
2102 2102
2103 2103 dep = NULL;
2104 2104 mp = modp->mod_mp;
2105 2105 mp->flags |= KOBJ_PRIM;
2106 2106
2107 2107 /* Bind new module to its dependents */
2108 2108 if (mp->depends_on != NULL && (dep =
2109 2109 mod_already_loaded(mp->depends_on)) == NULL) {
2110 2110 #ifdef KOBJ_DEBUG
2111 2111 if (kobj_debug & D_DEBUG) {
2112 2112 _kobj_printf(ops, "krtld: failed to resolve deps "
2113 2113 "for primary %s\n", modp->mod_modname);
2114 2114 }
2115 2115 #endif
2116 2116 return (-1);
2117 2117 }
2118 2118
2119 2119 if (dep != NULL)
2120 2120 add_dependent(mp, dep->mod_mp);
2121 2121
2122 2122 /*
2123 2123 * Relocate it. This module may not be part of a link map, so we
2124 2124 * can't use bind_primary.
2125 2125 */
2126 2126 if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2127 2127 do_relocations(mp) < 0) {
2128 2128 #ifdef KOBJ_DEBUG
2129 2129 if (kobj_debug & D_DEBUG) {
2130 2130 _kobj_printf(ops, "krtld: failed to relocate "
2131 2131 "primary %s\n", modp->mod_modname);
2132 2132 }
2133 2133 #endif
2134 2134 return (-1);
2135 2135 }
2136 2136
2137 2137 return (0);
2138 2138 }
2139 2139
2140 2140 static void
2141 2141 module_assign(struct modctl *cp, struct module *mp)
2142 2142 {
2143 2143 if (standalone) {
2144 2144 cp->mod_mp = mp;
2145 2145 return;
2146 2146 }
2147 2147 mutex_enter(&mod_lock);
2148 2148 cp->mod_mp = mp;
2149 2149 cp->mod_gencount++;
2150 2150 mutex_exit(&mod_lock);
2151 2151 }
2152 2152
2153 2153 void
2154 2154 kobj_unload_module(struct modctl *modp)
2155 2155 {
2156 2156 struct module *mp = modp->mod_mp;
2157 2157
2158 2158 if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2159 2159 _kobj_printf(ops, "text for %s ", mp->filename);
2160 2160 _kobj_printf(ops, "was at %p\n", mp->text);
2161 2161 mp->text = NULL; /* don't actually free it */
2162 2162 }
2163 2163
2164 2164 kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2165 2165
2166 2166 /*
2167 2167 * Null out mod_mp first, so consumers (debuggers) know not to look
2168 2168 * at the module structure any more.
2169 2169 */
2170 2170 mutex_enter(&mod_lock);
2171 2171 modp->mod_mp = NULL;
|
↓ open down ↓ |
2171 lines elided |
↑ open up ↑ |
2172 2172 mutex_exit(&mod_lock);
2173 2173
2174 2174 kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2175 2175 free_module_data(mp);
2176 2176 }
2177 2177
2178 2178 static void
2179 2179 free_module_data(struct module *mp)
2180 2180 {
2181 2181 struct module_list *lp, *tmp;
2182 + hotinline_desc_t *hid, *next;
2182 2183 int ksyms_exported = 0;
2183 2184
2184 2185 lp = mp->head;
2185 2186 while (lp) {
2186 2187 tmp = lp;
2187 2188 lp = lp->next;
2188 2189 kobj_free((char *)tmp, sizeof (*tmp));
2189 2190 }
2190 2191
2192 + /* release hotinlines */
2193 + hid = mp->hi_calls;
2194 + while (hid != NULL) {
2195 + next = hid->hid_next;
2196 + kobj_free(hid->hid_symname, strlen(hid->hid_symname) + 1);
2197 + kobj_free(hid, sizeof (hotinline_desc_t));
2198 + hid = next;
2199 + }
2200 +
2191 2201 rw_enter(&ksyms_lock, RW_WRITER);
2192 2202 if (mp->symspace) {
2193 2203 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2194 2204 vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2195 2205 ksyms_exported = 1;
2196 2206 } else {
2197 2207 if (mp->flags & KOBJ_NOKSYMS)
2198 2208 ksyms_exported = 1;
2199 2209 kobj_free(mp->symspace, mp->symsize);
2200 2210 }
2201 2211 }
2202 2212 rw_exit(&ksyms_lock);
2203 2213
2204 2214 if (mp->ctfdata) {
2205 2215 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2206 2216 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2207 2217 else
2208 2218 kobj_free(mp->ctfdata, mp->ctfsize);
2209 2219 }
2210 2220
2211 2221 if (mp->sigdata)
2212 2222 kobj_free(mp->sigdata, mp->sigsize);
2213 2223
2214 2224 /*
2215 2225 * We did not get far enough into kobj_export_ksyms() to free allocated
2216 2226 * buffers because we encounted error conditions. Free the buffers.
2217 2227 */
2218 2228 if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2219 2229 uint_t shn;
2220 2230 Shdr *shp;
2221 2231
2222 2232 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2223 2233 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2224 2234 switch (shp->sh_type) {
2225 2235 case SHT_RELA:
2226 2236 case SHT_REL:
2227 2237 if (shp->sh_addr != 0)
2228 2238 kobj_free((void *)shp->sh_addr,
2229 2239 shp->sh_size);
2230 2240 break;
2231 2241 }
2232 2242 }
2233 2243 err_free_done:
2234 2244 if (!(mp->flags & KOBJ_PRIM)) {
2235 2245 kobj_free(mp->shdrs,
2236 2246 mp->hdr.e_shentsize * mp->hdr.e_shnum);
2237 2247 }
2238 2248 }
2239 2249
2240 2250 if (mp->bss)
2241 2251 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2242 2252
2243 2253 if (mp->fbt_tab)
2244 2254 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2245 2255
2246 2256 if (mp->textwin_base)
2247 2257 kobj_textwin_free(mp);
2248 2258
2249 2259 if (mp->sdt_probes != NULL) {
2250 2260 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2251 2261
2252 2262 while (sdp != NULL) {
2253 2263 next = sdp->sdpd_next;
2254 2264 kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2255 2265 kobj_free(sdp, sizeof (sdt_probedesc_t));
2256 2266 sdp = next;
2257 2267 }
2258 2268 }
2259 2269
2260 2270 if (mp->sdt_tab)
2261 2271 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2262 2272 if (mp->text)
2263 2273 vmem_free(text_arena, mp->text, mp->text_size);
2264 2274 if (mp->data)
2265 2275 vmem_free(data_arena, mp->data, mp->data_size);
2266 2276 if (mp->depends_on)
2267 2277 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2268 2278 if (mp->filename)
2269 2279 kobj_free(mp->filename, strlen(mp->filename)+1);
2270 2280
2271 2281 kobj_free((char *)mp, sizeof (*mp));
2272 2282 }
2273 2283
2274 2284 static int
2275 2285 get_progbits(struct module *mp, struct _buf *file)
2276 2286 {
2277 2287 struct proginfo *tp, *dp, *sdp;
2278 2288 Shdr *shp;
2279 2289 reloc_dest_t dest = NULL;
2280 2290 uintptr_t bits_ptr;
2281 2291 uintptr_t text = 0, data, textptr;
2282 2292 uint_t shn;
2283 2293 int err = -1;
2284 2294
2285 2295 tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2286 2296 dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2287 2297 sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2288 2298 /*
2289 2299 * loop through sections to find out how much space we need
2290 2300 * for text, data, (also bss that is already assigned)
2291 2301 */
2292 2302 if (get_progbits_size(mp, tp, dp, sdp) < 0)
2293 2303 goto done;
2294 2304
2295 2305 mp->text_size = tp->size;
2296 2306 mp->data_size = dp->size;
2297 2307
2298 2308 if (standalone) {
2299 2309 caddr_t limit = _data;
2300 2310
2301 2311 if (lg_pagesize && _text + lg_pagesize < limit)
2302 2312 limit = _text + lg_pagesize;
2303 2313
2304 2314 mp->text = kobj_segbrk(&_etext, mp->text_size,
2305 2315 tp->align, limit);
2306 2316 /*
2307 2317 * If we can't grow the text segment, try the
2308 2318 * data segment before failing.
2309 2319 */
2310 2320 if (mp->text == NULL) {
2311 2321 mp->text = kobj_segbrk(&_edata, mp->text_size,
2312 2322 tp->align, 0);
2313 2323 }
2314 2324
2315 2325 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2316 2326
2317 2327 if (mp->text == NULL || mp->data == NULL)
2318 2328 goto done;
2319 2329
2320 2330 } else {
2321 2331 if (text_arena == NULL)
2322 2332 kobj_vmem_init(&text_arena, &data_arena);
2323 2333
2324 2334 /*
2325 2335 * some architectures may want to load the module on a
2326 2336 * page that is currently read only. It may not be
2327 2337 * possible for those architectures to remap their page
2328 2338 * on the fly. So we provide a facility for them to hang
2329 2339 * a private hook where the memory they assign the module
2330 2340 * is not the actual place where the module loads.
2331 2341 *
2332 2342 * In this case there are two addresses that deal with the
2333 2343 * modload.
2334 2344 * 1) the final destination of the module
2335 2345 * 2) the address that is used to view the newly
2336 2346 * loaded module until all the relocations relative to 1
2337 2347 * above are completed.
2338 2348 *
2339 2349 * That is what dest is used for below.
2340 2350 */
2341 2351 mp->text_size += tp->align;
2342 2352 mp->data_size += dp->align;
2343 2353
2344 2354 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2345 2355
2346 2356 /*
2347 2357 * a remap is taking place. Align the text ptr relative
2348 2358 * to the secondary mapping. That is where the bits will
2349 2359 * be read in.
2350 2360 */
2351 2361 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2352 2362 mp->text, mp->text_size)) {
2353 2363 off_t off = (uintptr_t)mp->text & PAGEOFFSET;
2354 2364 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2355 2365 caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2356 2366 caddr_t orig = mp->text - off;
2357 2367 pgcnt_t pages = size / PAGESIZE;
2358 2368
2359 2369 dest = (reloc_dest_t)(map + off);
2360 2370 text = ALIGN((uintptr_t)dest, tp->align);
2361 2371
2362 2372 while (pages--) {
2363 2373 hat_devload(kas.a_hat, map, PAGESIZE,
2364 2374 hat_getpfnum(kas.a_hat, orig),
2365 2375 PROT_READ | PROT_WRITE | PROT_EXEC,
2366 2376 HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2367 2377 map += PAGESIZE;
2368 2378 orig += PAGESIZE;
2369 2379 }
2370 2380 /*
2371 2381 * Since we set up a non-cacheable mapping, we need
2372 2382 * to flush any old entries in the cache that might
2373 2383 * be left around from the read-only mapping.
2374 2384 */
2375 2385 dcache_flushall();
2376 2386 }
2377 2387 if (mp->data_size)
2378 2388 mp->data = vmem_alloc(data_arena, mp->data_size,
2379 2389 VM_SLEEP | VM_BESTFIT);
2380 2390 }
2381 2391 textptr = (uintptr_t)mp->text;
2382 2392 textptr = ALIGN(textptr, tp->align);
2383 2393 mp->destination = dest;
2384 2394
2385 2395 /*
2386 2396 * This is the case where a remap is not being done.
2387 2397 */
2388 2398 if (text == 0)
2389 2399 text = ALIGN((uintptr_t)mp->text, tp->align);
2390 2400 data = ALIGN((uintptr_t)mp->data, dp->align);
2391 2401
2392 2402 /* now loop though sections assigning addresses and loading the data */
2393 2403 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2394 2404 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2395 2405 if (!(shp->sh_flags & SHF_ALLOC))
2396 2406 continue;
2397 2407
2398 2408 if ((shp->sh_flags & SHF_WRITE) == 0)
2399 2409 bits_ptr = text;
2400 2410 else
2401 2411 bits_ptr = data;
2402 2412
2403 2413 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2404 2414
2405 2415 if (shp->sh_type == SHT_NOBITS) {
2406 2416 /*
2407 2417 * Zero bss.
2408 2418 */
2409 2419 bzero((caddr_t)bits_ptr, shp->sh_size);
2410 2420 shp->sh_type = SHT_PROGBITS;
2411 2421 } else {
2412 2422 if (kobj_read_file(file, (char *)bits_ptr,
2413 2423 shp->sh_size, shp->sh_offset) < 0)
2414 2424 goto done;
2415 2425 }
2416 2426
2417 2427 if (shp->sh_flags & SHF_WRITE) {
2418 2428 shp->sh_addr = bits_ptr;
2419 2429 } else {
2420 2430 textptr = ALIGN(textptr, shp->sh_addralign);
2421 2431 shp->sh_addr = textptr;
2422 2432 textptr += shp->sh_size;
2423 2433 }
2424 2434
2425 2435 bits_ptr += shp->sh_size;
2426 2436 if ((shp->sh_flags & SHF_WRITE) == 0)
2427 2437 text = bits_ptr;
2428 2438 else
2429 2439 data = bits_ptr;
2430 2440 }
2431 2441
2432 2442 err = 0;
2433 2443 done:
2434 2444 /*
2435 2445 * Free and mark as freed the section headers here so that
2436 2446 * free_module_data() does not have to worry about this buffer.
2437 2447 *
2438 2448 * This buffer is freed here because one of the possible reasons
2439 2449 * for error is a section with non-zero sh_addr and in that case
2440 2450 * free_module_data() would have no way of recognizing that this
2441 2451 * buffer was unallocated.
2442 2452 */
2443 2453 if (err != 0) {
2444 2454 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2445 2455 mp->shdrs = NULL;
2446 2456 }
2447 2457
2448 2458 (void) kobj_free(tp, sizeof (struct proginfo));
2449 2459 (void) kobj_free(dp, sizeof (struct proginfo));
2450 2460 (void) kobj_free(sdp, sizeof (struct proginfo));
2451 2461
2452 2462 return (err);
2453 2463 }
2454 2464
2455 2465 /*
2456 2466 * Go through suppress_sym_list to see if "multiply defined"
2457 2467 * warning of this symbol should be suppressed. Return 1 if
2458 2468 * warning should be suppressed, 0 otherwise.
2459 2469 */
2460 2470 static int
2461 2471 kobj_suppress_warning(char *symname)
2462 2472 {
2463 2473 int i;
2464 2474
2465 2475 for (i = 0; suppress_sym_list[i] != NULL; i++) {
2466 2476 if (strcmp(suppress_sym_list[i], symname) == 0)
2467 2477 return (1);
2468 2478 }
2469 2479
2470 2480 return (0);
2471 2481 }
2472 2482
2473 2483 static int
2474 2484 get_syms(struct module *mp, struct _buf *file)
2475 2485 {
2476 2486 uint_t shn;
2477 2487 Shdr *shp;
2478 2488 uint_t i;
2479 2489 Sym *sp, *ksp;
2480 2490 char *symname;
2481 2491 int dosymtab = 0;
2482 2492
2483 2493 /*
2484 2494 * Find the interesting sections.
2485 2495 */
2486 2496 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2487 2497 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2488 2498 switch (shp->sh_type) {
2489 2499 case SHT_SYMTAB:
2490 2500 mp->symtbl_section = shn;
2491 2501 mp->symhdr = shp;
2492 2502 dosymtab++;
2493 2503 break;
2494 2504
2495 2505 case SHT_RELA:
2496 2506 case SHT_REL:
2497 2507 /*
2498 2508 * Already loaded.
2499 2509 */
2500 2510 if (shp->sh_addr)
2501 2511 continue;
2502 2512
2503 2513 /* KM_TMP since kobj_free'd in do_relocations */
2504 2514 shp->sh_addr = (Addr)
2505 2515 kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2506 2516
2507 2517 if (kobj_read_file(file, (char *)shp->sh_addr,
2508 2518 shp->sh_size, shp->sh_offset) < 0) {
2509 2519 _kobj_printf(ops, "krtld: get_syms: %s, ",
2510 2520 mp->filename);
2511 2521 _kobj_printf(ops, "error reading section %d\n",
2512 2522 shn);
2513 2523 return (-1);
2514 2524 }
2515 2525 break;
2516 2526 }
2517 2527 }
2518 2528
2519 2529 /*
2520 2530 * This is true for a stripped executable. In the case of
2521 2531 * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2522 2532 * and since that symbol information is still present everything
2523 2533 * is just fine.
2524 2534 */
2525 2535 if (!dosymtab) {
2526 2536 if (mp->flags & KOBJ_EXEC)
2527 2537 return (0);
2528 2538 _kobj_printf(ops, "krtld: get_syms: %s ",
2529 2539 mp->filename);
2530 2540 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2531 2541 return (-1);
2532 2542 }
2533 2543
2534 2544 /*
2535 2545 * get the associated string table header
2536 2546 */
2537 2547 if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2538 2548 return (-1);
2539 2549 mp->strhdr = (Shdr *)
2540 2550 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2541 2551
2542 2552 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2543 2553 mp->hashsize = kobj_gethashsize(mp->nsyms);
2544 2554
2545 2555 /*
2546 2556 * Allocate space for the symbol table, buckets, chains, and strings.
2547 2557 */
2548 2558 mp->symsize = mp->symhdr->sh_size +
2549 2559 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2550 2560 mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2551 2561
2552 2562 mp->symtbl = mp->symspace;
2553 2563 mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2554 2564 mp->chains = mp->buckets + mp->hashsize;
2555 2565 mp->strings = (char *)(mp->chains + mp->nsyms);
2556 2566
2557 2567 if (kobj_read_file(file, mp->symtbl,
2558 2568 mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2559 2569 kobj_read_file(file, mp->strings,
2560 2570 mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2561 2571 return (-1);
2562 2572
2563 2573 /*
2564 2574 * loop through the symbol table adjusting values to account
2565 2575 * for where each section got loaded into memory. Also
2566 2576 * fill in the hash table.
2567 2577 */
2568 2578 for (i = 1; i < mp->nsyms; i++) {
2569 2579 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2570 2580 if (sp->st_shndx < SHN_LORESERVE) {
2571 2581 if (sp->st_shndx >= mp->hdr.e_shnum) {
2572 2582 _kobj_printf(ops, "%s bad shndx ",
2573 2583 file->_name);
2574 2584 _kobj_printf(ops, "in symbol %d\n", i);
2575 2585 return (-1);
2576 2586 }
2577 2587 shp = (Shdr *)
2578 2588 (mp->shdrs +
2579 2589 sp->st_shndx * mp->hdr.e_shentsize);
2580 2590 if (!(mp->flags & KOBJ_EXEC))
2581 2591 sp->st_value += shp->sh_addr;
2582 2592 }
2583 2593
2584 2594 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2585 2595 continue;
2586 2596 if (sp->st_name >= mp->strhdr->sh_size)
2587 2597 return (-1);
2588 2598
2589 2599 symname = mp->strings + sp->st_name;
2590 2600
2591 2601 if (!(mp->flags & KOBJ_EXEC) &&
2592 2602 ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2593 2603 ksp = kobj_lookup_all(mp, symname, 0);
2594 2604
2595 2605 if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2596 2606 !kobj_suppress_warning(symname) &&
2597 2607 sp->st_shndx != SHN_UNDEF &&
2598 2608 sp->st_shndx != SHN_COMMON &&
2599 2609 ksp->st_shndx != SHN_UNDEF &&
2600 2610 ksp->st_shndx != SHN_COMMON) {
2601 2611 /*
2602 2612 * Unless this symbol is a stub, it's multiply
2603 2613 * defined. Multiply-defined symbols are
2604 2614 * usually bad, but some objects (kmdb) have
2605 2615 * a legitimate need to have their own
2606 2616 * copies of common functions.
2607 2617 */
2608 2618 if ((standalone ||
2609 2619 ksp->st_value < (uintptr_t)stubs_base ||
2610 2620 ksp->st_value >= (uintptr_t)stubs_end) &&
2611 2621 !(mp->flags & KOBJ_IGNMULDEF)) {
2612 2622 _kobj_printf(ops,
2613 2623 "%s symbol ", file->_name);
2614 2624 _kobj_printf(ops,
2615 2625 "%s multiply defined\n", symname);
2616 2626 }
2617 2627 }
2618 2628 }
2619 2629
2620 2630 sym_insert(mp, symname, i);
2621 2631 }
2622 2632
2623 2633 return (0);
2624 2634 }
2625 2635
2626 2636 static int
2627 2637 get_ctf(struct module *mp, struct _buf *file)
2628 2638 {
2629 2639 char *shstrtab, *ctfdata;
2630 2640 size_t shstrlen;
2631 2641 Shdr *shp;
2632 2642 uint_t i;
2633 2643
2634 2644 if (_moddebug & MODDEBUG_NOCTF)
2635 2645 return (0); /* do not attempt to even load CTF data */
2636 2646
2637 2647 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2638 2648 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2639 2649 mp->filename);
2640 2650 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2641 2651 mp->hdr.e_shstrndx);
2642 2652 return (-1);
2643 2653 }
2644 2654
2645 2655 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2646 2656 shstrlen = shp->sh_size;
2647 2657 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2648 2658
2649 2659 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2650 2660 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2651 2661 mp->filename);
2652 2662 _kobj_printf(ops, "error reading section %u\n",
2653 2663 mp->hdr.e_shstrndx);
2654 2664 kobj_free(shstrtab, shstrlen);
2655 2665 return (-1);
2656 2666 }
2657 2667
2658 2668 for (i = 0; i < mp->hdr.e_shnum; i++) {
2659 2669 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2660 2670
2661 2671 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2662 2672 strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2663 2673 ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2664 2674
2665 2675 if (kobj_read_file(file, ctfdata, shp->sh_size,
2666 2676 shp->sh_offset) < 0) {
2667 2677 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2668 2678 "reading .SUNW_ctf data\n", mp->filename);
2669 2679 kobj_free(ctfdata, shp->sh_size);
2670 2680 kobj_free(shstrtab, shstrlen);
2671 2681 return (-1);
2672 2682 }
2673 2683
2674 2684 mp->ctfdata = ctfdata;
2675 2685 mp->ctfsize = shp->sh_size;
2676 2686 break;
2677 2687 }
2678 2688 }
2679 2689
2680 2690 kobj_free(shstrtab, shstrlen);
2681 2691 return (0);
2682 2692 }
2683 2693
2684 2694 #define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */
2685 2695
2686 2696 /*
2687 2697 * Return the hash of the ELF sections that are memory resident.
2688 2698 * i.e. text and data. We skip a SHT_NOBITS section since it occupies
2689 2699 * no space in the file. We use SHA1 here since libelfsign uses
2690 2700 * it and both places need to use the same algorithm.
2691 2701 */
2692 2702 static void
2693 2703 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2694 2704 {
2695 2705 uint_t shn;
2696 2706 Shdr *shp;
2697 2707 SHA1_CTX ctx;
2698 2708
2699 2709 SHA1Init(&ctx);
2700 2710
2701 2711 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2702 2712 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2703 2713 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2704 2714 continue;
2705 2715
2706 2716 /*
2707 2717 * The check should ideally be shp->sh_type == SHT_NOBITS.
2708 2718 * However, we can't do that check here as get_progbits()
2709 2719 * resets the type.
2710 2720 */
2711 2721 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2712 2722 continue;
2713 2723 #ifdef KOBJ_DEBUG
2714 2724 if (kobj_debug & D_DEBUG)
2715 2725 _kobj_printf(ops,
2716 2726 "krtld: crypto_es_hash: updating hash with"
2717 2727 " %s data size=%lx\n", shstrtab + shp->sh_name,
2718 2728 (size_t)shp->sh_size);
2719 2729 #endif
2720 2730 ASSERT(shp->sh_addr != 0);
2721 2731 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2722 2732 }
2723 2733
2724 2734 SHA1Final((uchar_t *)hash, &ctx);
2725 2735 }
2726 2736
2727 2737 /*
2728 2738 * Get the .SUNW_signature section for the module, it it exists.
2729 2739 *
2730 2740 * This section exists only for crypto modules. None of the
2731 2741 * primary modules have this section currently.
2732 2742 */
2733 2743 static void
2734 2744 get_signature(struct module *mp, struct _buf *file)
2735 2745 {
2736 2746 char *shstrtab, *sigdata = NULL;
2737 2747 size_t shstrlen;
2738 2748 Shdr *shp;
2739 2749 uint_t i;
2740 2750
2741 2751 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2742 2752 _kobj_printf(ops, "krtld: get_signature: %s, ",
2743 2753 mp->filename);
2744 2754 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2745 2755 mp->hdr.e_shstrndx);
2746 2756 return;
2747 2757 }
2748 2758
2749 2759 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2750 2760 shstrlen = shp->sh_size;
2751 2761 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2752 2762
2753 2763 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2754 2764 _kobj_printf(ops, "krtld: get_signature: %s, ",
2755 2765 mp->filename);
2756 2766 _kobj_printf(ops, "error reading section %u\n",
2757 2767 mp->hdr.e_shstrndx);
2758 2768 kobj_free(shstrtab, shstrlen);
2759 2769 return;
2760 2770 }
2761 2771
2762 2772 for (i = 0; i < mp->hdr.e_shnum; i++) {
2763 2773 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2764 2774 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2765 2775 strcmp(shstrtab + shp->sh_name,
2766 2776 ELF_SIGNATURE_SECTION) == 0) {
2767 2777 filesig_vers_t filesig_version;
2768 2778 size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2769 2779 sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2770 2780
2771 2781 if (kobj_read_file(file, sigdata, shp->sh_size,
2772 2782 shp->sh_offset) < 0) {
2773 2783 _kobj_printf(ops, "krtld: get_signature: %s,"
2774 2784 " error reading .SUNW_signature data\n",
2775 2785 mp->filename);
2776 2786 kobj_free(sigdata, sigsize);
2777 2787 kobj_free(shstrtab, shstrlen);
2778 2788 return;
2779 2789 }
2780 2790 filesig_version = ((struct filesignatures *)sigdata)->
2781 2791 filesig_sig.filesig_version;
2782 2792 if (!(filesig_version == FILESIG_VERSION1 ||
2783 2793 filesig_version == FILESIG_VERSION3)) {
2784 2794 /* skip versions we don't understand */
2785 2795 kobj_free(sigdata, sigsize);
2786 2796 kobj_free(shstrtab, shstrlen);
2787 2797 return;
2788 2798 }
2789 2799
2790 2800 mp->sigdata = sigdata;
2791 2801 mp->sigsize = sigsize;
2792 2802 break;
2793 2803 }
2794 2804 }
2795 2805
2796 2806 if (sigdata != NULL) {
2797 2807 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2798 2808 }
2799 2809
2800 2810 kobj_free(shstrtab, shstrlen);
2801 2811 }
2802 2812
2803 2813 static void
2804 2814 add_dependent(struct module *mp, struct module *dep)
2805 2815 {
2806 2816 struct module_list *lp;
2807 2817
2808 2818 for (lp = mp->head; lp; lp = lp->next) {
2809 2819 if (lp->mp == dep)
2810 2820 return; /* already on the list */
2811 2821 }
2812 2822
2813 2823 if (lp == NULL) {
2814 2824 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2815 2825
2816 2826 lp->mp = dep;
2817 2827 lp->next = NULL;
2818 2828 if (mp->tail)
2819 2829 mp->tail->next = lp;
2820 2830 else
2821 2831 mp->head = lp;
2822 2832 mp->tail = lp;
2823 2833 }
2824 2834 }
2825 2835
2826 2836 static int
2827 2837 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2828 2838 {
2829 2839 struct module *mp;
2830 2840 struct modctl *req;
2831 2841 char *d, *p, *q;
2832 2842 int c;
2833 2843 char *err_modname = NULL;
2834 2844
2835 2845 mp = modp->mod_mp;
2836 2846
2837 2847 if ((p = mp->depends_on) == NULL)
2838 2848 return (0);
2839 2849
2840 2850 for (;;) {
2841 2851 /*
2842 2852 * Skip space.
2843 2853 */
2844 2854 while (*p && (*p == ' ' || *p == '\t'))
2845 2855 p++;
2846 2856 /*
2847 2857 * Get module name.
2848 2858 */
2849 2859 d = p;
2850 2860 q = modname;
2851 2861 c = 0;
2852 2862 while (*p && *p != ' ' && *p != '\t') {
2853 2863 if (c < modnamelen - 1) {
2854 2864 *q++ = *p;
2855 2865 c++;
2856 2866 }
2857 2867 p++;
2858 2868 }
2859 2869
2860 2870 if (q == modname)
2861 2871 break;
2862 2872
2863 2873 if (c == modnamelen - 1) {
2864 2874 char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2865 2875
2866 2876 (void) strncpy(dep, d, p - d + 1);
2867 2877 dep[p - d] = '\0';
2868 2878
2869 2879 _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2870 2880 _kobj_printf(ops, "'%s' too long ", dep);
2871 2881 _kobj_printf(ops, "(max %d chars)\n", (int)modnamelen);
2872 2882
2873 2883 kobj_free(dep, p - d + 1);
2874 2884
2875 2885 return (-1);
2876 2886 }
2877 2887
2878 2888 *q = '\0';
2879 2889 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2880 2890 #ifndef KOBJ_DEBUG
2881 2891 if (_moddebug & MODDEBUG_LOADMSG) {
2882 2892 #endif /* KOBJ_DEBUG */
2883 2893 _kobj_printf(ops,
2884 2894 "%s: unable to resolve dependency, ",
2885 2895 modp->mod_modname);
2886 2896 _kobj_printf(ops, "cannot load module '%s'\n",
2887 2897 modname);
2888 2898 #ifndef KOBJ_DEBUG
2889 2899 }
2890 2900 #endif /* KOBJ_DEBUG */
2891 2901 if (err_modname == NULL) {
2892 2902 /*
2893 2903 * This must be the same size as the modname
2894 2904 * one.
2895 2905 */
2896 2906 err_modname = kobj_zalloc(MODMAXNAMELEN,
2897 2907 KM_WAIT);
2898 2908
2899 2909 /*
2900 2910 * We can use strcpy() here without fearing
2901 2911 * the NULL terminator because the size of
2902 2912 * err_modname is the same as one of modname,
2903 2913 * and it's filled with zeros.
2904 2914 */
2905 2915 (void) strcpy(err_modname, modname);
2906 2916 }
2907 2917 continue;
2908 2918 }
2909 2919
2910 2920 add_dependent(mp, req->mod_mp);
2911 2921 mod_release_mod(req);
2912 2922
2913 2923 }
2914 2924
2915 2925 if (err_modname != NULL) {
2916 2926 /*
2917 2927 * Copy the first module name where you detect an error to keep
2918 2928 * its behavior the same as before.
2919 2929 * This way keeps minimizing the memory use for error
2920 2930 * modules, and this might be important at boot time because
2921 2931 * the memory usage is a crucial factor for booting in most
2922 2932 * cases. You can expect more verbose messages when using
2923 2933 * a debug kernel or setting a bit in moddebug.
2924 2934 */
2925 2935 bzero(modname, MODMAXNAMELEN);
2926 2936 (void) strcpy(modname, err_modname);
2927 2937 kobj_free(err_modname, MODMAXNAMELEN);
2928 2938 return (-1);
2929 2939 }
2930 2940
2931 2941 return (0);
2932 2942 }
2933 2943
2934 2944 static int
2935 2945 do_common(struct module *mp)
2936 2946 {
2937 2947 int err;
2938 2948
2939 2949 /*
2940 2950 * first time through, assign all symbols defined in other
2941 2951 * modules, and count up how much common space will be needed
2942 2952 * (bss_size and bss_align)
2943 2953 */
2944 2954 if ((err = do_symbols(mp, 0)) < 0)
2945 2955 return (err);
2946 2956 /*
2947 2957 * increase bss_size by the maximum delta that could be
2948 2958 * computed by the ALIGN below
2949 2959 */
2950 2960 mp->bss_size += mp->bss_align;
2951 2961 if (mp->bss_size) {
2952 2962 if (standalone)
2953 2963 mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2954 2964 MINALIGN, 0);
2955 2965 else
2956 2966 mp->bss = (uintptr_t)vmem_alloc(data_arena,
2957 2967 mp->bss_size, VM_SLEEP | VM_BESTFIT);
2958 2968 bzero((void *)mp->bss, mp->bss_size);
2959 2969 /* now assign addresses to all common symbols */
2960 2970 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2961 2971 return (err);
2962 2972 }
2963 2973 return (0);
2964 2974 }
2965 2975
2966 2976 static int
2967 2977 do_symbols(struct module *mp, Elf64_Addr bss_base)
2968 2978 {
2969 2979 int bss_align;
2970 2980 uintptr_t bss_ptr;
2971 2981 int err;
2972 2982 int i;
2973 2983 Sym *sp, *sp1;
2974 2984 char *name;
2975 2985 int assign;
2976 2986 int resolved = 1;
2977 2987
2978 2988 /*
2979 2989 * Nothing left to do (optimization).
2980 2990 */
2981 2991 if (mp->flags & KOBJ_RESOLVED)
2982 2992 return (0);
2983 2993
2984 2994 assign = (bss_base) ? 1 : 0;
2985 2995 bss_ptr = bss_base;
2986 2996 bss_align = 0;
2987 2997 err = 0;
2988 2998
2989 2999 for (i = 1; i < mp->nsyms; i++) {
2990 3000 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
2991 3001 /*
2992 3002 * we know that st_name is in bounds, since get_sections
2993 3003 * has already checked all of the symbols
2994 3004 */
2995 3005 name = mp->strings + sp->st_name;
2996 3006 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
2997 3007 continue;
2998 3008 #if defined(__sparc)
2999 3009 /*
3000 3010 * Register symbols are ignored in the kernel
3001 3011 */
3002 3012 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER) {
3003 3013 if (*name != '\0') {
3004 3014 _kobj_printf(ops, "%s: named REGISTER symbol ",
3005 3015 mp->filename);
3006 3016 _kobj_printf(ops, "not supported '%s'\n",
3007 3017 name);
3008 3018 err = DOSYM_UNDEF;
3009 3019 }
3010 3020 continue;
3011 3021 }
3012 3022 #endif /* __sparc */
3013 3023 /*
3014 3024 * TLS symbols are ignored in the kernel
3015 3025 */
3016 3026 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
3017 3027 _kobj_printf(ops, "%s: TLS symbol ",
3018 3028 mp->filename);
3019 3029 _kobj_printf(ops, "not supported '%s'\n",
3020 3030 name);
3021 3031 err = DOSYM_UNDEF;
3022 3032 continue;
3023 3033 }
3024 3034
3025 3035 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
|
↓ open down ↓ |
825 lines elided |
↑ open up ↑ |
3026 3036 if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
3027 3037 sp->st_shndx = SHN_ABS;
3028 3038 sp->st_value = sp1->st_value;
3029 3039 continue;
3030 3040 }
3031 3041 }
3032 3042
3033 3043 if (sp->st_shndx == SHN_UNDEF) {
3034 3044 resolved = 0;
3035 3045
3046 + /*
3047 + * Skip over sdt probes and smap calls,
3048 + * they're relocated later.
3049 + */
3036 3050 if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
3037 3051 continue;
3052 +#if defined(__x86)
3053 + if (strcmp(name, "smap_enable") == 0 ||
3054 + strcmp(name, "smap_disable") == 0)
3055 + continue;
3056 +#endif /* defined(__x86) */
3038 3057
3058 +
3039 3059 /*
3040 3060 * If it's not a weak reference and it's
3041 3061 * not a primary object, it's an error.
3042 3062 * (Primary objects may take more than
3043 3063 * one pass to resolve)
3044 3064 */
3045 3065 if (!(mp->flags & KOBJ_PRIM) &&
3046 3066 ELF_ST_BIND(sp->st_info) != STB_WEAK) {
3047 3067 _kobj_printf(ops, "%s: undefined symbol",
3048 3068 mp->filename);
3049 3069 _kobj_printf(ops, " '%s'\n", name);
3050 3070 /*
3051 3071 * Try to determine whether this symbol
3052 3072 * represents a dependency on obsolete
3053 3073 * unsafe driver support. This is just
3054 3074 * to make the warning more informative.
3055 3075 */
3056 3076 if (strcmp(name, "sleep") == 0 ||
3057 3077 strcmp(name, "unsleep") == 0 ||
3058 3078 strcmp(name, "wakeup") == 0 ||
3059 3079 strcmp(name, "bsd_compat_ioctl") == 0 ||
3060 3080 strcmp(name, "unsafe_driver") == 0 ||
3061 3081 strncmp(name, "spl", 3) == 0 ||
3062 3082 strncmp(name, "i_ddi_spl", 9) == 0)
3063 3083 err = DOSYM_UNSAFE;
3064 3084 if (err == 0)
3065 3085 err = DOSYM_UNDEF;
3066 3086 }
3067 3087 continue;
3068 3088 }
3069 3089 /*
3070 3090 * It's a common symbol - st_value is the
3071 3091 * required alignment.
3072 3092 */
3073 3093 if (sp->st_value > bss_align)
3074 3094 bss_align = sp->st_value;
3075 3095 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3076 3096 if (assign) {
3077 3097 sp->st_shndx = SHN_ABS;
3078 3098 sp->st_value = bss_ptr;
3079 3099 }
3080 3100 bss_ptr += sp->st_size;
3081 3101 }
3082 3102 if (err)
3083 3103 return (err);
3084 3104 if (assign == 0 && mp->bss == 0) {
3085 3105 mp->bss_align = bss_align;
3086 3106 mp->bss_size = bss_ptr;
3087 3107 } else if (resolved) {
3088 3108 mp->flags |= KOBJ_RESOLVED;
3089 3109 }
3090 3110
3091 3111 return (0);
3092 3112 }
3093 3113
3094 3114 uint_t
3095 3115 kobj_hash_name(const char *p)
3096 3116 {
3097 3117 uint_t g;
3098 3118 uint_t hval;
3099 3119
3100 3120 hval = 0;
3101 3121 while (*p) {
3102 3122 hval = (hval << 4) + *p++;
3103 3123 if ((g = (hval & 0xf0000000)) != 0)
3104 3124 hval ^= g >> 24;
3105 3125 hval &= ~g;
3106 3126 }
3107 3127 return (hval);
3108 3128 }
3109 3129
3110 3130 /* look for name in all modules */
3111 3131 uintptr_t
3112 3132 kobj_getsymvalue(char *name, int kernelonly)
3113 3133 {
3114 3134 Sym *sp;
3115 3135 struct modctl *modp;
3116 3136 struct module *mp;
3117 3137 uintptr_t value = 0;
3118 3138
3119 3139 if ((sp = kobj_lookup_kernel(name)) != NULL)
3120 3140 return ((uintptr_t)sp->st_value);
3121 3141
3122 3142 if (kernelonly)
3123 3143 return (0); /* didn't find it in the kernel so give up */
3124 3144
3125 3145 mutex_enter(&mod_lock);
3126 3146 modp = &modules;
3127 3147 do {
3128 3148 mp = (struct module *)modp->mod_mp;
3129 3149 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3130 3150 (sp = lookup_one(mp, name))) {
3131 3151 value = (uintptr_t)sp->st_value;
3132 3152 break;
3133 3153 }
3134 3154 } while ((modp = modp->mod_next) != &modules);
3135 3155 mutex_exit(&mod_lock);
3136 3156 return (value);
3137 3157 }
3138 3158
3139 3159 /* look for a symbol near value. */
3140 3160 char *
3141 3161 kobj_getsymname(uintptr_t value, ulong_t *offset)
3142 3162 {
3143 3163 char *name = NULL;
3144 3164 struct modctl *modp;
3145 3165
3146 3166 struct modctl_list *lp;
3147 3167 struct module *mp;
3148 3168
3149 3169 /*
3150 3170 * Loop through the primary kernel modules.
3151 3171 */
3152 3172 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3153 3173 mp = mod(lp);
3154 3174
3155 3175 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3156 3176 return (name);
3157 3177 }
3158 3178
3159 3179 mutex_enter(&mod_lock);
3160 3180 modp = &modules;
3161 3181 do {
3162 3182 mp = (struct module *)modp->mod_mp;
3163 3183 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3164 3184 (name = kobj_searchsym(mp, value, offset)))
3165 3185 break;
3166 3186 } while ((modp = modp->mod_next) != &modules);
3167 3187 mutex_exit(&mod_lock);
3168 3188 return (name);
3169 3189 }
3170 3190
3171 3191 /* return address of symbol and size */
3172 3192
3173 3193 uintptr_t
3174 3194 kobj_getelfsym(char *name, void *mp, int *size)
3175 3195 {
3176 3196 Sym *sp;
3177 3197
3178 3198 if (mp == NULL)
3179 3199 sp = kobj_lookup_kernel(name);
3180 3200 else
3181 3201 sp = lookup_one(mp, name);
3182 3202
3183 3203 if (sp == NULL)
3184 3204 return (0);
3185 3205
3186 3206 *size = (int)sp->st_size;
3187 3207 return ((uintptr_t)sp->st_value);
3188 3208 }
3189 3209
3190 3210 uintptr_t
3191 3211 kobj_lookup(struct module *mod, const char *name)
3192 3212 {
3193 3213 Sym *sp;
3194 3214
3195 3215 sp = lookup_one(mod, name);
3196 3216
3197 3217 if (sp == NULL)
3198 3218 return (0);
3199 3219
3200 3220 return ((uintptr_t)sp->st_value);
3201 3221 }
3202 3222
3203 3223 char *
3204 3224 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3205 3225 {
3206 3226 Sym *symtabptr;
3207 3227 char *strtabptr;
3208 3228 int symnum;
3209 3229 Sym *sym;
3210 3230 Sym *cursym;
3211 3231 uintptr_t curval;
3212 3232
3213 3233 *offset = (ulong_t)-1l; /* assume not found */
3214 3234 cursym = NULL;
3215 3235
3216 3236 if (kobj_addrcheck(mp, (void *)value) != 0)
3217 3237 return (NULL); /* not in this module */
3218 3238
3219 3239 strtabptr = mp->strings;
3220 3240 symtabptr = (Sym *)mp->symtbl;
3221 3241
3222 3242 /*
3223 3243 * Scan the module's symbol table for a symbol <= value
3224 3244 */
3225 3245 for (symnum = 1, sym = symtabptr + 1;
3226 3246 symnum < mp->nsyms; symnum++, sym = (Sym *)
3227 3247 ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3228 3248 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3229 3249 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3230 3250 continue;
3231 3251 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3232 3252 ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3233 3253 continue;
3234 3254 }
3235 3255
3236 3256 curval = (uintptr_t)sym->st_value;
3237 3257
3238 3258 if (curval > value)
3239 3259 continue;
3240 3260
3241 3261 /*
3242 3262 * If one or both are functions...
3243 3263 */
3244 3264 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3245 3265 ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3246 3266 /* Ignore if the address is out of the bounds */
3247 3267 if (value - sym->st_value >= sym->st_size)
3248 3268 continue;
3249 3269
3250 3270 if (cursym != NULL &&
3251 3271 ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3252 3272 /* Prefer the function to the non-function */
3253 3273 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3254 3274 continue;
3255 3275
3256 3276 /* Prefer the larger of the two functions */
3257 3277 if (sym->st_size <= cursym->st_size)
3258 3278 continue;
3259 3279 }
3260 3280 } else if (value - curval >= *offset) {
3261 3281 continue;
3262 3282 }
3263 3283
3264 3284 *offset = (ulong_t)(value - curval);
3265 3285 cursym = sym;
3266 3286 }
3267 3287 if (cursym == NULL)
3268 3288 return (NULL);
3269 3289
3270 3290 return (strtabptr + cursym->st_name);
3271 3291 }
3272 3292
3273 3293 Sym *
3274 3294 kobj_lookup_all(struct module *mp, char *name, int include_self)
3275 3295 {
3276 3296 Sym *sp;
3277 3297 struct module_list *mlp;
3278 3298 struct modctl_list *clp;
3279 3299 struct module *mmp;
3280 3300
3281 3301 if (include_self && (sp = lookup_one(mp, name)) != NULL)
3282 3302 return (sp);
3283 3303
3284 3304 for (mlp = mp->head; mlp; mlp = mlp->next) {
3285 3305 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3286 3306 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3287 3307 return (sp);
3288 3308 }
3289 3309
3290 3310 /*
3291 3311 * Loop through the primary kernel modules.
3292 3312 */
3293 3313 for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3294 3314 mmp = mod(clp);
3295 3315
3296 3316 if (mmp == NULL || mp == mmp)
3297 3317 continue;
3298 3318
3299 3319 if ((sp = lookup_one(mmp, name)) != NULL &&
3300 3320 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3301 3321 return (sp);
3302 3322 }
3303 3323 return (NULL);
3304 3324 }
3305 3325
3306 3326 Sym *
3307 3327 kobj_lookup_kernel(const char *name)
3308 3328 {
3309 3329 struct modctl_list *lp;
3310 3330 struct module *mp;
3311 3331 Sym *sp;
3312 3332
3313 3333 /*
3314 3334 * Loop through the primary kernel modules.
3315 3335 */
3316 3336 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3317 3337 mp = mod(lp);
3318 3338
3319 3339 if (mp == NULL)
3320 3340 continue;
3321 3341
3322 3342 if ((sp = lookup_one(mp, name)) != NULL)
3323 3343 return (sp);
3324 3344 }
3325 3345 return (NULL);
3326 3346 }
3327 3347
3328 3348 static Sym *
3329 3349 lookup_one(struct module *mp, const char *name)
3330 3350 {
3331 3351 symid_t *ip;
3332 3352 char *name1;
3333 3353 Sym *sp;
3334 3354
3335 3355 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3336 3356 ip = &mp->chains[*ip]) {
3337 3357 sp = (Sym *)(mp->symtbl +
3338 3358 mp->symhdr->sh_entsize * *ip);
3339 3359 name1 = mp->strings + sp->st_name;
3340 3360 if (strcmp(name, name1) == 0 &&
3341 3361 ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3342 3362 sp->st_shndx != SHN_UNDEF &&
3343 3363 sp->st_shndx != SHN_COMMON)
3344 3364 return (sp);
3345 3365 }
3346 3366 return (NULL);
3347 3367 }
3348 3368
3349 3369 /*
3350 3370 * Lookup a given symbol pointer in the module's symbol hash. If the symbol
3351 3371 * is hashed, return the symbol pointer; otherwise return NULL.
3352 3372 */
3353 3373 static Sym *
3354 3374 sym_lookup(struct module *mp, Sym *ksp)
3355 3375 {
3356 3376 char *name = mp->strings + ksp->st_name;
3357 3377 symid_t *ip;
3358 3378 Sym *sp;
3359 3379
3360 3380 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3361 3381 ip = &mp->chains[*ip]) {
3362 3382 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3363 3383 if (sp == ksp)
3364 3384 return (ksp);
3365 3385 }
3366 3386 return (NULL);
3367 3387 }
3368 3388
3369 3389 static void
3370 3390 sym_insert(struct module *mp, char *name, symid_t index)
3371 3391 {
3372 3392 symid_t *ip;
3373 3393
3374 3394 #ifdef KOBJ_DEBUG
3375 3395 if (kobj_debug & D_SYMBOLS) {
3376 3396 static struct module *lastmp = NULL;
3377 3397 Sym *sp;
3378 3398 if (lastmp != mp) {
3379 3399 _kobj_printf(ops,
3380 3400 "krtld: symbol entry: file=%s\n",
3381 3401 mp->filename);
3382 3402 _kobj_printf(ops,
3383 3403 "krtld:\tsymndx\tvalue\t\t"
3384 3404 "symbol name\n");
3385 3405 lastmp = mp;
3386 3406 }
3387 3407 sp = (Sym *)(mp->symtbl +
3388 3408 index * mp->symhdr->sh_entsize);
3389 3409 _kobj_printf(ops, "krtld:\t[%3d]", index);
3390 3410 _kobj_printf(ops, "\t0x%lx", sp->st_value);
3391 3411 _kobj_printf(ops, "\t%s\n", name);
3392 3412 }
3393 3413 #endif
3394 3414
3395 3415 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3396 3416 ip = &mp->chains[*ip]) {
3397 3417 ;
3398 3418 }
3399 3419 *ip = index;
3400 3420 }
3401 3421
3402 3422 struct modctl *
3403 3423 kobj_boot_mod_lookup(const char *modname)
3404 3424 {
3405 3425 struct modctl *mctl = kobj_modules;
3406 3426
3407 3427 do {
3408 3428 if (strcmp(modname, mctl->mod_modname) == 0)
3409 3429 return (mctl);
3410 3430 } while ((mctl = mctl->mod_next) != kobj_modules);
3411 3431
3412 3432 return (NULL);
3413 3433 }
3414 3434
3415 3435 /*
3416 3436 * Determine if the module exists.
3417 3437 */
3418 3438 int
3419 3439 kobj_path_exists(char *name, int use_path)
3420 3440 {
3421 3441 struct _buf *file;
3422 3442
3423 3443 file = kobj_open_path(name, use_path, 1);
3424 3444 #ifdef MODDIR_SUFFIX
3425 3445 if (file == (struct _buf *)-1)
3426 3446 file = kobj_open_path(name, use_path, 0);
3427 3447 #endif /* MODDIR_SUFFIX */
3428 3448 if (file == (struct _buf *)-1)
3429 3449 return (0);
3430 3450 kobj_close_file(file);
3431 3451 return (1);
3432 3452 }
3433 3453
3434 3454 /*
3435 3455 * fullname is dynamically allocated to be able to hold the
3436 3456 * maximum size string that can be constructed from name.
3437 3457 * path is exactly like the shell PATH variable.
3438 3458 */
3439 3459 struct _buf *
3440 3460 kobj_open_path(char *name, int use_path, int use_moddir_suffix)
3441 3461 {
3442 3462 char *p, *q;
3443 3463 char *pathp;
3444 3464 char *pathpsave;
3445 3465 char *fullname;
3446 3466 int maxpathlen;
3447 3467 struct _buf *file;
3448 3468
3449 3469 #if !defined(MODDIR_SUFFIX)
3450 3470 use_moddir_suffix = B_FALSE;
3451 3471 #endif
3452 3472
3453 3473 if (!use_path)
3454 3474 pathp = ""; /* use name as specified */
3455 3475 else
3456 3476 pathp = kobj_module_path;
3457 3477 /* use configured default path */
3458 3478
3459 3479 pathpsave = pathp; /* keep this for error reporting */
3460 3480
3461 3481 /*
3462 3482 * Allocate enough space for the largest possible fullname.
3463 3483 * since path is of the form <directory> : <directory> : ...
3464 3484 * we're potentially allocating a little more than we need to
3465 3485 * but we'll allocate the exact amount when we find the right directory.
3466 3486 * (The + 3 below is one for NULL terminator and one for the '/'
3467 3487 * we might have to add at the beginning of path and one for
3468 3488 * the '/' between path and name.)
3469 3489 */
3470 3490 maxpathlen = strlen(pathp) + strlen(name) + 3;
3471 3491 /* sizeof includes null */
3472 3492 maxpathlen += sizeof (slash_moddir_suffix_slash) - 1;
3473 3493 fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3474 3494
3475 3495 for (;;) {
3476 3496 p = fullname;
3477 3497 if (*pathp != '\0' && *pathp != '/')
3478 3498 *p++ = '/'; /* path must start with '/' */
3479 3499 while (*pathp && *pathp != ':' && *pathp != ' ')
3480 3500 *p++ = *pathp++;
3481 3501 if (p != fullname && p[-1] != '/')
3482 3502 *p++ = '/';
3483 3503 if (use_moddir_suffix) {
3484 3504 char *b = basename(name);
3485 3505 char *s;
3486 3506
3487 3507 /* copy everything up to the base name */
3488 3508 q = name;
3489 3509 while (q != b && *q)
3490 3510 *p++ = *q++;
3491 3511 s = slash_moddir_suffix_slash;
3492 3512 while (*s)
3493 3513 *p++ = *s++;
3494 3514 /* copy the rest */
3495 3515 while (*b)
3496 3516 *p++ = *b++;
3497 3517 } else {
3498 3518 q = name;
3499 3519 while (*q)
3500 3520 *p++ = *q++;
3501 3521 }
3502 3522 *p = 0;
3503 3523 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3504 3524 kobj_free(fullname, maxpathlen);
3505 3525 return (file);
3506 3526 }
3507 3527 while (*pathp == ' ' || *pathp == ':')
3508 3528 pathp++;
3509 3529 if (*pathp == 0)
3510 3530 break;
3511 3531
3512 3532 }
3513 3533 kobj_free(fullname, maxpathlen);
3514 3534 if (_moddebug & MODDEBUG_ERRMSG) {
3515 3535 _kobj_printf(ops, "can't open %s,", name);
3516 3536 _kobj_printf(ops, " path is %s\n", pathpsave);
3517 3537 }
3518 3538 return ((struct _buf *)-1);
3519 3539 }
3520 3540
3521 3541 intptr_t
3522 3542 kobj_open(char *filename)
3523 3543 {
3524 3544 struct vnode *vp;
3525 3545 int fd;
3526 3546
3527 3547 if (_modrootloaded) {
3528 3548 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3529 3549 int Errno;
3530 3550
3531 3551 /*
3532 3552 * Hand off the open to a thread who has a
3533 3553 * stack size capable handling the request.
3534 3554 */
3535 3555 if (curthread != &t0) {
3536 3556 (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3537 3557 kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3538 3558 sema_p(<p->sema);
3539 3559 Errno = ltp->Errno;
3540 3560 vp = ltp->vp;
3541 3561 } else {
3542 3562 /*
3543 3563 * 1098067: module creds should not be those of the
3544 3564 * caller
3545 3565 */
3546 3566 cred_t *saved_cred = curthread->t_cred;
3547 3567 curthread->t_cred = kcred;
3548 3568 Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3549 3569 0, 0, rootdir, -1);
3550 3570 curthread->t_cred = saved_cred;
3551 3571 }
3552 3572 kobjopen_free(ltp);
3553 3573
3554 3574 if (Errno) {
3555 3575 if (_moddebug & MODDEBUG_ERRMSG) {
3556 3576 _kobj_printf(ops,
3557 3577 "kobj_open: vn_open of %s fails, ",
3558 3578 filename);
3559 3579 _kobj_printf(ops, "Errno = %d\n", Errno);
3560 3580 }
3561 3581 return (-1);
3562 3582 } else {
3563 3583 if (_moddebug & MODDEBUG_ERRMSG) {
3564 3584 _kobj_printf(ops, "kobj_open: '%s'", filename);
3565 3585 _kobj_printf(ops, " vp = %p\n", vp);
3566 3586 }
3567 3587 return ((intptr_t)vp);
3568 3588 }
3569 3589 } else {
3570 3590 fd = kobj_boot_open(filename, 0);
3571 3591
3572 3592 if (_moddebug & MODDEBUG_ERRMSG) {
3573 3593 if (fd < 0)
3574 3594 _kobj_printf(ops,
3575 3595 "kobj_open: can't open %s\n", filename);
3576 3596 else {
3577 3597 _kobj_printf(ops, "kobj_open: '%s'", filename);
3578 3598 _kobj_printf(ops, " descr = 0x%x\n", fd);
3579 3599 }
3580 3600 }
3581 3601 return ((intptr_t)fd);
3582 3602 }
3583 3603 }
3584 3604
3585 3605 /*
3586 3606 * Calls to kobj_open() are handled off to this routine as a separate thread.
3587 3607 */
3588 3608 static void
3589 3609 kobjopen_thread(struct kobjopen_tctl *ltp)
3590 3610 {
3591 3611 kmutex_t cpr_lk;
3592 3612 callb_cpr_t cpr_i;
3593 3613
3594 3614 mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3595 3615 CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3596 3616 ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3597 3617 0, 0);
3598 3618 sema_v(<p->sema);
3599 3619 mutex_enter(&cpr_lk);
3600 3620 CALLB_CPR_EXIT(&cpr_i);
3601 3621 mutex_destroy(&cpr_lk);
3602 3622 thread_exit();
3603 3623 }
3604 3624
3605 3625 /*
3606 3626 * allocate and initialize a kobjopen thread structure
3607 3627 */
3608 3628 static struct kobjopen_tctl *
3609 3629 kobjopen_alloc(char *filename)
3610 3630 {
3611 3631 struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3612 3632
3613 3633 ASSERT(filename != NULL);
3614 3634
3615 3635 ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3616 3636 bcopy(filename, ltp->name, strlen(filename) + 1);
3617 3637 sema_init(<p->sema, 0, NULL, SEMA_DEFAULT, NULL);
3618 3638 return (ltp);
3619 3639 }
3620 3640
3621 3641 /*
3622 3642 * free a kobjopen thread control structure
3623 3643 */
3624 3644 static void
3625 3645 kobjopen_free(struct kobjopen_tctl *ltp)
3626 3646 {
3627 3647 sema_destroy(<p->sema);
3628 3648 kmem_free(ltp->name, strlen(ltp->name) + 1);
3629 3649 kmem_free(ltp, sizeof (*ltp));
3630 3650 }
3631 3651
3632 3652 int
3633 3653 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3634 3654 {
3635 3655 int stat;
3636 3656 ssize_t resid;
3637 3657
3638 3658 if (_modrootloaded) {
3639 3659 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3640 3660 (offset_t)offset, UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3641 3661 &resid)) != 0) {
3642 3662 _kobj_printf(ops,
3643 3663 "vn_rdwr failed with error 0x%x\n", stat);
3644 3664 return (-1);
3645 3665 }
3646 3666 return (size - resid);
3647 3667 } else {
3648 3668 int count = 0;
3649 3669
3650 3670 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3651 3671 _kobj_printf(ops,
3652 3672 "kobj_read: seek 0x%x failed\n", offset);
3653 3673 return (-1);
3654 3674 }
3655 3675
3656 3676 count = kobj_boot_read((int)descr, buf, size);
3657 3677 if (count < size) {
3658 3678 if (_moddebug & MODDEBUG_ERRMSG) {
3659 3679 _kobj_printf(ops,
3660 3680 "kobj_read: req %d bytes, ", size);
3661 3681 _kobj_printf(ops, "got %d\n", count);
3662 3682 }
3663 3683 }
3664 3684 return (count);
3665 3685 }
3666 3686 }
3667 3687
3668 3688 void
3669 3689 kobj_close(intptr_t descr)
3670 3690 {
3671 3691 if (_moddebug & MODDEBUG_ERRMSG)
3672 3692 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3673 3693
3674 3694 if (_modrootloaded) {
3675 3695 struct vnode *vp = (struct vnode *)descr;
3676 3696 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
3677 3697 VN_RELE(vp);
3678 3698 } else
3679 3699 (void) kobj_boot_close((int)descr);
3680 3700 }
3681 3701
3682 3702 int
3683 3703 kobj_fstat(intptr_t descr, struct bootstat *buf)
3684 3704 {
3685 3705 if (buf == NULL)
3686 3706 return (-1);
3687 3707
3688 3708 if (_modrootloaded) {
3689 3709 vattr_t vattr;
3690 3710 struct vnode *vp = (struct vnode *)descr;
3691 3711 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3692 3712 return (-1);
3693 3713
3694 3714 /*
3695 3715 * The vattr and bootstat structures are similar, but not
3696 3716 * identical. We do our best to fill in the bootstat structure
3697 3717 * from the contents of vattr (transfering only the ones that
3698 3718 * are obvious.
3699 3719 */
3700 3720
3701 3721 buf->st_mode = (uint32_t)vattr.va_mode;
3702 3722 buf->st_nlink = (uint32_t)vattr.va_nlink;
3703 3723 buf->st_uid = (int32_t)vattr.va_uid;
3704 3724 buf->st_gid = (int32_t)vattr.va_gid;
3705 3725 buf->st_rdev = (uint64_t)vattr.va_rdev;
3706 3726 buf->st_size = (uint64_t)vattr.va_size;
3707 3727 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3708 3728 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3709 3729 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3710 3730 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3711 3731 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3712 3732 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3713 3733 buf->st_blksize = (int32_t)vattr.va_blksize;
3714 3734 buf->st_blocks = (int64_t)vattr.va_nblocks;
3715 3735
3716 3736 return (0);
3717 3737 }
3718 3738
3719 3739 return (kobj_boot_fstat((int)descr, buf));
3720 3740 }
3721 3741
3722 3742
3723 3743 struct _buf *
3724 3744 kobj_open_file(char *name)
3725 3745 {
3726 3746 struct _buf *file;
3727 3747 struct compinfo cbuf;
3728 3748 intptr_t fd;
3729 3749
3730 3750 if ((fd = kobj_open(name)) == -1) {
3731 3751 return ((struct _buf *)-1);
3732 3752 }
3733 3753
3734 3754 file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3735 3755 file->_fd = fd;
3736 3756 file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3737 3757 file->_cnt = file->_size = file->_off = 0;
3738 3758 file->_ln = 1;
3739 3759 file->_ptr = file->_base;
3740 3760 (void) strcpy(file->_name, name);
3741 3761
3742 3762 /*
3743 3763 * Before root is mounted, we must check
3744 3764 * for a compressed file and do our own
3745 3765 * buffering.
3746 3766 */
3747 3767 if (_modrootloaded) {
3748 3768 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3749 3769 file->_bsize = MAXBSIZE;
3750 3770
3751 3771 /* Check if the file is compressed */
3752 3772 file->_iscmp = kobj_is_compressed(fd);
3753 3773 } else {
3754 3774 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3755 3775 kobj_close_file(file);
3756 3776 return ((struct _buf *)-1);
3757 3777 }
3758 3778 file->_iscmp = cbuf.iscmp;
3759 3779 if (file->_iscmp) {
3760 3780 if (kobj_comp_setup(file, &cbuf) != 0) {
3761 3781 kobj_close_file(file);
3762 3782 return ((struct _buf *)-1);
3763 3783 }
3764 3784 } else {
3765 3785 file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3766 3786 file->_bsize = cbuf.blksize;
3767 3787 }
3768 3788 }
3769 3789 return (file);
3770 3790 }
3771 3791
3772 3792 static int
3773 3793 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3774 3794 {
3775 3795 struct comphdr *hdr;
3776 3796
3777 3797 /*
3778 3798 * read the compressed image into memory,
3779 3799 * so we can deompress from there
3780 3800 */
3781 3801 file->_dsize = cip->fsize;
3782 3802 file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3783 3803 if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3784 3804 kobj_free(file->_dbuf, cip->fsize);
3785 3805 return (-1);
3786 3806 }
3787 3807
3788 3808 hdr = kobj_comphdr(file);
3789 3809 if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3790 3810 hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3791 3811 !ISP2(hdr->ch_blksize)) {
3792 3812 kobj_free(file->_dbuf, cip->fsize);
3793 3813 return (-1);
3794 3814 }
3795 3815 file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3796 3816 file->_bsize = hdr->ch_blksize;
3797 3817 return (0);
3798 3818 }
3799 3819
3800 3820 void
3801 3821 kobj_close_file(struct _buf *file)
3802 3822 {
3803 3823 kobj_close(file->_fd);
3804 3824 if (file->_base != NULL)
3805 3825 kobj_free(file->_base, file->_bsize);
3806 3826 if (file->_dbuf != NULL)
3807 3827 kobj_free(file->_dbuf, file->_dsize);
3808 3828 kobj_free(file->_name, strlen(file->_name)+1);
3809 3829 kobj_free(file, sizeof (struct _buf));
3810 3830 }
3811 3831
3812 3832 int
3813 3833 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3814 3834 {
3815 3835 int b_size, c_size;
3816 3836 int b_off; /* Offset into buffer for start of bcopy */
3817 3837 int count = 0;
3818 3838 int page_addr;
3819 3839
3820 3840 if (_moddebug & MODDEBUG_ERRMSG) {
3821 3841 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3822 3842 _kobj_printf(ops, " offset=%x at", off);
3823 3843 _kobj_printf(ops, " buf=%lx\n", (uintptr_t)buf);
3824 3844 }
3825 3845
3826 3846 /*
3827 3847 * Handle compressed (gzip for now) file here. First get the
3828 3848 * compressed size, then read the image into memory and finally
3829 3849 * call zlib to decompress the image at the supplied memory buffer.
3830 3850 */
3831 3851 if (file->_iscmp == CH_MAGIC_GZIP) {
3832 3852 ulong_t dlen;
3833 3853 vattr_t vattr;
3834 3854 struct vnode *vp = (struct vnode *)file->_fd;
3835 3855 ssize_t resid;
3836 3856 int err = 0;
3837 3857
3838 3858 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3839 3859 return (-1);
3840 3860
3841 3861 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3842 3862 file->_dsize = vattr.va_size;
3843 3863
3844 3864 /* Read the compressed file into memory */
3845 3865 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3846 3866 (offset_t)(0), UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3847 3867 &resid)) != 0) {
3848 3868
3849 3869 _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3850 3870 "error code 0x%x\n", err);
3851 3871 return (-1);
3852 3872 }
3853 3873
3854 3874 dlen = size;
3855 3875
3856 3876 /* Decompress the image at the supplied memory buffer */
3857 3877 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3858 3878 vattr.va_size)) != Z_OK) {
3859 3879 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3860 3880 "failed, error code : 0x%x\n", err);
3861 3881 return (-1);
3862 3882 }
3863 3883
3864 3884 if (dlen != size) {
3865 3885 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3866 3886 "failed to uncompress (size returned 0x%lx , "
3867 3887 "expected size: 0x%x)\n", dlen, size);
3868 3888 return (-1);
3869 3889 }
3870 3890
3871 3891 return (0);
3872 3892 }
3873 3893
3874 3894 while (size) {
3875 3895 page_addr = F_PAGE(file, off);
3876 3896 b_size = file->_size;
3877 3897 /*
3878 3898 * If we have the filesystem page the caller's referring to
3879 3899 * and we have something in the buffer,
3880 3900 * satisfy as much of the request from the buffer as we can.
3881 3901 */
3882 3902 if (page_addr == file->_off && b_size > 0) {
3883 3903 b_off = B_OFFSET(file, off);
3884 3904 c_size = b_size - b_off;
3885 3905 /*
3886 3906 * If there's nothing to copy, we're at EOF.
3887 3907 */
3888 3908 if (c_size <= 0)
3889 3909 break;
3890 3910 if (c_size > size)
3891 3911 c_size = size;
3892 3912 if (buf) {
3893 3913 if (_moddebug & MODDEBUG_ERRMSG)
3894 3914 _kobj_printf(ops, "copying %x bytes\n",
3895 3915 c_size);
3896 3916 bcopy(file->_base+b_off, buf, c_size);
3897 3917 size -= c_size;
3898 3918 off += c_size;
3899 3919 buf += c_size;
3900 3920 count += c_size;
3901 3921 } else {
3902 3922 _kobj_printf(ops, "kobj_read: system error");
3903 3923 count = -1;
3904 3924 break;
3905 3925 }
3906 3926 } else {
3907 3927 /*
3908 3928 * If the caller's offset is page aligned and
3909 3929 * the caller want's at least a filesystem page and
3910 3930 * the caller provided a buffer,
3911 3931 * read directly into the caller's buffer.
3912 3932 */
3913 3933 if (page_addr == off &&
3914 3934 (c_size = F_BLKS(file, size)) && buf) {
3915 3935 c_size = kobj_read_blks(file, buf, c_size,
3916 3936 page_addr);
3917 3937 if (c_size < 0) {
3918 3938 count = -1;
3919 3939 break;
3920 3940 }
3921 3941 count += c_size;
3922 3942 if (c_size != F_BLKS(file, size))
3923 3943 break;
3924 3944 size -= c_size;
3925 3945 off += c_size;
3926 3946 buf += c_size;
3927 3947 /*
3928 3948 * Otherwise, read into our buffer and copy next time
3929 3949 * around the loop.
3930 3950 */
3931 3951 } else {
3932 3952 file->_off = page_addr;
3933 3953 c_size = kobj_read_blks(file, file->_base,
3934 3954 file->_bsize, page_addr);
3935 3955 file->_ptr = file->_base;
3936 3956 file->_cnt = c_size;
3937 3957 file->_size = c_size;
3938 3958 /*
3939 3959 * If a _filbuf call or nothing read, break.
3940 3960 */
3941 3961 if (buf == NULL || c_size <= 0) {
3942 3962 count = c_size;
3943 3963 break;
3944 3964 }
3945 3965 }
3946 3966 if (_moddebug & MODDEBUG_ERRMSG)
3947 3967 _kobj_printf(ops, "read %x bytes\n", c_size);
3948 3968 }
3949 3969 }
3950 3970 if (_moddebug & MODDEBUG_ERRMSG)
3951 3971 _kobj_printf(ops, "count = %x\n", count);
3952 3972
3953 3973 return (count);
3954 3974 }
3955 3975
3956 3976 static int
3957 3977 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3958 3978 {
3959 3979 int ret;
3960 3980
3961 3981 ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3962 3982 if (file->_iscmp) {
3963 3983 uint_t blks;
3964 3984 int nret;
3965 3985
3966 3986 ret = 0;
3967 3987 for (blks = size / file->_bsize; blks != 0; blks--) {
3968 3988 nret = kobj_uncomp_blk(file, buf, off);
3969 3989 if (nret == -1)
3970 3990 return (-1);
3971 3991 buf += nret;
3972 3992 off += nret;
3973 3993 ret += nret;
3974 3994 if (nret < file->_bsize)
3975 3995 break;
3976 3996 }
3977 3997 } else
3978 3998 ret = kobj_read(file->_fd, buf, size, off);
3979 3999 return (ret);
3980 4000 }
3981 4001
3982 4002 static int
3983 4003 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
3984 4004 {
3985 4005 struct comphdr *hdr = kobj_comphdr(file);
3986 4006 ulong_t dlen, slen;
3987 4007 caddr_t src;
3988 4008 int i;
3989 4009
3990 4010 dlen = file->_bsize;
3991 4011 i = off / file->_bsize;
3992 4012 src = file->_dbuf + hdr->ch_blkmap[i];
3993 4013 if (i == hdr->ch_fsize / file->_bsize)
3994 4014 slen = file->_dsize - hdr->ch_blkmap[i];
3995 4015 else
3996 4016 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
3997 4017 if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
3998 4018 return (-1);
3999 4019 return (dlen);
4000 4020 }
4001 4021
4002 4022 int
4003 4023 kobj_filbuf(struct _buf *f)
4004 4024 {
4005 4025 if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
4006 4026 return (kobj_getc(f));
4007 4027 return (-1);
4008 4028 }
4009 4029
4010 4030 void
4011 4031 kobj_free(void *address, size_t size)
4012 4032 {
4013 4033 if (standalone)
4014 4034 return;
4015 4035
4016 4036 kmem_free(address, size);
4017 4037 kobj_stat.nfree_calls++;
4018 4038 kobj_stat.nfree += size;
4019 4039 }
4020 4040
4021 4041 void *
4022 4042 kobj_zalloc(size_t size, int flag)
4023 4043 {
4024 4044 void *v;
4025 4045
4026 4046 if ((v = kobj_alloc(size, flag)) != 0) {
4027 4047 bzero(v, size);
4028 4048 }
4029 4049
4030 4050 return (v);
4031 4051 }
4032 4052
4033 4053 void *
4034 4054 kobj_alloc(size_t size, int flag)
4035 4055 {
4036 4056 /*
4037 4057 * If we are running standalone in the
4038 4058 * linker, we ask boot for memory.
4039 4059 * Either it's temporary memory that we lose
4040 4060 * once boot is mapped out or we allocate it
4041 4061 * permanently using the dynamic data segment.
4042 4062 */
4043 4063 if (standalone) {
4044 4064 #if defined(_OBP)
4045 4065 if (flag & (KM_TMP | KM_SCRATCH))
4046 4066 return (bop_temp_alloc(size, MINALIGN));
4047 4067 #else
4048 4068 if (flag & (KM_TMP | KM_SCRATCH))
4049 4069 return (BOP_ALLOC(ops, 0, size, MINALIGN));
4050 4070 #endif
4051 4071 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
4052 4072 }
4053 4073
4054 4074 kobj_stat.nalloc_calls++;
4055 4075 kobj_stat.nalloc += size;
4056 4076
4057 4077 return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
4058 4078 }
4059 4079
4060 4080 /*
4061 4081 * Allow the "mod" system to sync up with the work
4062 4082 * already done by kobj during the initial loading
4063 4083 * of the kernel. This also gives us a chance
4064 4084 * to reallocate memory that belongs to boot.
4065 4085 */
4066 4086 void
4067 4087 kobj_sync(void)
4068 4088 {
4069 4089 struct modctl_list *lp, **lpp;
4070 4090
4071 4091 /*
4072 4092 * The module path can be set in /etc/system via 'moddir' commands
4073 4093 */
4074 4094 if (default_path != NULL)
4075 4095 kobj_module_path = default_path;
4076 4096 else
4077 4097 default_path = kobj_module_path;
4078 4098
4079 4099 ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
4080 4100 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4081 4101
4082 4102 ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
4083 4103 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4084 4104
4085 4105 /*
4086 4106 * Move symbol tables from boot memory to ksyms_arena.
4087 4107 */
4088 4108 for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
4089 4109 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
4090 4110 kobj_export_module(mod(lp));
4091 4111 }
4092 4112 }
4093 4113
4094 4114 caddr_t
4095 4115 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
4096 4116 {
4097 4117 uintptr_t va, pva;
4098 4118 size_t alloc_pgsz = kobj_mmu_pagesize;
4099 4119 size_t alloc_align = BO_NO_ALIGN;
4100 4120 size_t alloc_size;
4101 4121
4102 4122 /*
4103 4123 * If we are using "large" mappings for the kernel,
4104 4124 * request aligned memory from boot using the
4105 4125 * "large" pagesize.
4106 4126 */
4107 4127 if (lg_pagesize) {
4108 4128 alloc_align = lg_pagesize;
4109 4129 alloc_pgsz = lg_pagesize;
4110 4130 }
4111 4131
4112 4132 #if defined(__sparc)
4113 4133 /* account for redzone */
4114 4134 if (limit)
4115 4135 limit -= alloc_pgsz;
4116 4136 #endif /* __sparc */
4117 4137
4118 4138 va = ALIGN((uintptr_t)*spp, align);
4119 4139 pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4120 4140 /*
4121 4141 * Need more pages?
4122 4142 */
4123 4143 if (va + size > pva) {
4124 4144 uintptr_t npva;
4125 4145
4126 4146 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4127 4147 /*
4128 4148 * Check for overlapping segments.
4129 4149 */
4130 4150 if (limit && limit <= *spp + alloc_size) {
4131 4151 return ((caddr_t)0);
4132 4152 }
4133 4153
4134 4154 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4135 4155 alloc_size, alloc_align);
4136 4156
4137 4157 if (npva == 0) {
4138 4158 _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4139 4159 alloc_size);
4140 4160 _kobj_printf(ops, " aligned %lx", alloc_align);
4141 4161 _kobj_printf(ops, " at 0x%lx\n", pva);
4142 4162 return (NULL);
4143 4163 }
4144 4164 }
4145 4165 *spp = (caddr_t)(va + size);
4146 4166
4147 4167 return ((caddr_t)va);
4148 4168 }
4149 4169
4150 4170 /*
4151 4171 * Calculate the number of output hash buckets.
4152 4172 * We use the next prime larger than n / 4,
4153 4173 * so the average hash chain is about 4 entries.
4154 4174 * More buckets would just be a waste of memory.
4155 4175 */
4156 4176 uint_t
4157 4177 kobj_gethashsize(uint_t n)
4158 4178 {
4159 4179 int f;
4160 4180 int hsize = MAX(n / 4, 2);
4161 4181
4162 4182 for (f = 2; f * f <= hsize; f++)
4163 4183 if (hsize % f == 0)
4164 4184 hsize += f = 1;
4165 4185
4166 4186 return (hsize);
4167 4187 }
4168 4188
4169 4189 /*
4170 4190 * Get the file size.
4171 4191 *
4172 4192 * Before root is mounted, files are compressed in the boot_archive ramdisk
4173 4193 * (in the memory). kobj_fstat would return the compressed file size.
4174 4194 * In order to get the uncompressed file size, read the file to the end and
4175 4195 * count its size.
4176 4196 */
4177 4197 int
4178 4198 kobj_get_filesize(struct _buf *file, uint64_t *size)
4179 4199 {
4180 4200 int err = 0;
4181 4201 ssize_t resid;
4182 4202 uint32_t buf;
4183 4203
4184 4204 if (_modrootloaded) {
4185 4205 struct bootstat bst;
4186 4206
4187 4207 if (kobj_fstat(file->_fd, &bst) != 0)
4188 4208 return (EIO);
4189 4209 *size = bst.st_size;
4190 4210
4191 4211 if (file->_iscmp == CH_MAGIC_GZIP) {
4192 4212 /*
4193 4213 * Read the last 4 bytes of the compressed (gzip)
4194 4214 * image to get the size of its uncompressed
4195 4215 * version.
4196 4216 */
4197 4217 if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4198 4218 (char *)(&buf), 4, (offset_t)(*size - 4),
4199 4219 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid))
4200 4220 != 0) {
4201 4221 _kobj_printf(ops, "kobj_get_filesize: "
4202 4222 "vn_rdwr() failed with error 0x%x\n", err);
4203 4223 return (-1);
4204 4224 }
4205 4225
4206 4226 *size = (uint64_t)buf;
4207 4227 }
4208 4228 } else {
4209 4229
4210 4230 #if defined(_OBP)
4211 4231 struct bootstat bsb;
4212 4232
4213 4233 if (file->_iscmp) {
4214 4234 struct comphdr *hdr = kobj_comphdr(file);
4215 4235
4216 4236 *size = hdr->ch_fsize;
4217 4237 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4218 4238 return (EIO);
4219 4239 else
4220 4240 *size = bsb.st_size;
4221 4241 #else
4222 4242 char *buf;
4223 4243 int count;
4224 4244 uint64_t offset = 0;
4225 4245
4226 4246 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4227 4247 do {
4228 4248 count = kobj_read_file(file, buf, MAXBSIZE, offset);
4229 4249 if (count < 0) {
4230 4250 kmem_free(buf, MAXBSIZE);
4231 4251 return (EIO);
4232 4252 }
4233 4253 offset += count;
4234 4254 } while (count == MAXBSIZE);
4235 4255 kmem_free(buf, MAXBSIZE);
4236 4256
4237 4257 *size = offset;
4238 4258 #endif
4239 4259 }
4240 4260
4241 4261 return (0);
4242 4262 }
4243 4263
4244 4264 static char *
4245 4265 basename(char *s)
4246 4266 {
4247 4267 char *p, *q;
4248 4268
4249 4269 q = NULL;
4250 4270 p = s;
4251 4271 do {
4252 4272 if (*p == '/')
4253 4273 q = p;
4254 4274 } while (*p++);
4255 4275 return (q ? q + 1 : s);
4256 4276 }
4257 4277
4258 4278 void
4259 4279 kobj_stat_get(kobj_stat_t *kp)
4260 4280 {
4261 4281 *kp = kobj_stat;
4262 4282 }
4263 4283
4264 4284 int
4265 4285 kobj_getpagesize()
4266 4286 {
4267 4287 return (lg_pagesize);
4268 4288 }
4269 4289
4270 4290 void
4271 4291 kobj_textwin_alloc(struct module *mp)
4272 4292 {
4273 4293 ASSERT(MUTEX_HELD(&mod_lock));
4274 4294
4275 4295 if (mp->textwin != NULL)
4276 4296 return;
4277 4297
4278 4298 /*
4279 4299 * If the text is not contained in the heap, then it is not contained
4280 4300 * by a writable mapping. (Specifically, it's on the nucleus page.)
4281 4301 * We allocate a read/write mapping for this module's text to allow
4282 4302 * the text to be patched without calling hot_patch_kernel_text()
4283 4303 * (which is quite slow).
4284 4304 */
4285 4305 if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4286 4306 uintptr_t text = (uintptr_t)mp->text;
4287 4307 uintptr_t size = (uintptr_t)mp->text_size;
4288 4308 uintptr_t i;
4289 4309 caddr_t va;
4290 4310 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4291 4311 (text & PAGEMASK);
4292 4312
4293 4313 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4294 4314
4295 4315 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4296 4316 hat_devload(kas.a_hat, va, PAGESIZE,
4297 4317 hat_getpfnum(kas.a_hat, (caddr_t)i),
4298 4318 PROT_READ | PROT_WRITE,
4299 4319 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4300 4320 va += PAGESIZE;
4301 4321 }
4302 4322
4303 4323 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4304 4324 } else {
4305 4325 mp->textwin = mp->text;
4306 4326 }
4307 4327 }
4308 4328
4309 4329 void
4310 4330 kobj_textwin_free(struct module *mp)
4311 4331 {
4312 4332 uintptr_t text = (uintptr_t)mp->text;
4313 4333 uintptr_t tsize = (uintptr_t)mp->text_size;
4314 4334 size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4315 4335 (text & PAGEMASK));
4316 4336
4317 4337 mp->textwin = NULL;
4318 4338
4319 4339 if (mp->textwin_base == NULL)
4320 4340 return;
4321 4341
4322 4342 hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4323 4343 vmem_free(heap_arena, mp->textwin_base, size);
4324 4344 mp->textwin_base = NULL;
4325 4345 }
4326 4346
4327 4347 static char *
4328 4348 find_libmacro(char *name)
4329 4349 {
4330 4350 int lmi;
4331 4351
4332 4352 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4333 4353 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4334 4354 return (libmacros[lmi].lmi_list);
4335 4355 }
4336 4356 return (NULL);
4337 4357 }
4338 4358
4339 4359 /*
4340 4360 * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4341 4361 * returns path if successful, else NULL
4342 4362 * Support multiple $MACROs expansion and the first valid path will be returned
4343 4363 * Caller's responsibility to provide enough space in path to expand
4344 4364 */
4345 4365 char *
4346 4366 expand_libmacro(char *tail, char *path, char *pathend)
4347 4367 {
4348 4368 char c, *p, *p1, *p2, *path2, *endp;
4349 4369 int diff, lmi, macrolen, valid_macro, more_macro;
4350 4370 struct _buf *file;
4351 4371
4352 4372 /*
4353 4373 * check for $MACROS between nulls or slashes
4354 4374 */
4355 4375 p = strchr(tail, '$');
4356 4376 if (p == NULL)
4357 4377 return (NULL);
4358 4378 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4359 4379 macrolen = libmacros[lmi].lmi_macrolen;
4360 4380 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4361 4381 break;
4362 4382 }
4363 4383
4364 4384 valid_macro = 0;
4365 4385 if (lmi < NLIBMACROS) {
4366 4386 /*
4367 4387 * The following checks are used to restrict expansion of
4368 4388 * macros to those that form a full directory/file name
4369 4389 * and to keep the behavior same as before. If this
4370 4390 * restriction is removed or no longer valid in the future,
4371 4391 * the checks below can be deleted.
4372 4392 */
4373 4393 if ((p == tail) || (*(p - 1) == '/')) {
4374 4394 c = *(p + macrolen + 1);
4375 4395 if (c == '/' || c == '\0')
4376 4396 valid_macro = 1;
4377 4397 }
4378 4398 }
4379 4399
4380 4400 if (!valid_macro) {
4381 4401 p2 = strchr(p, '/');
4382 4402 /*
4383 4403 * if no more macro to expand, then just copy whatever left
4384 4404 * and check whether it exists
4385 4405 */
4386 4406 if (p2 == NULL || strchr(p2, '$') == NULL) {
4387 4407 (void) strcpy(pathend, tail);
4388 4408 if ((file = kobj_open_path(path, 1, 1)) !=
4389 4409 (struct _buf *)-1) {
4390 4410 kobj_close_file(file);
4391 4411 return (path);
4392 4412 } else
4393 4413 return (NULL);
4394 4414 } else {
4395 4415 /*
4396 4416 * copy all chars before '/' and call expand_libmacro()
4397 4417 * again
4398 4418 */
4399 4419 diff = p2 - tail;
4400 4420 bcopy(tail, pathend, diff);
4401 4421 pathend += diff;
4402 4422 *(pathend) = '\0';
4403 4423 return (expand_libmacro(p2, path, pathend));
4404 4424 }
4405 4425 }
4406 4426
4407 4427 more_macro = 0;
4408 4428 if (c != '\0') {
4409 4429 endp = p + macrolen + 1;
4410 4430 if (strchr(endp, '$') != NULL)
4411 4431 more_macro = 1;
4412 4432 } else
4413 4433 endp = NULL;
4414 4434
4415 4435 /*
4416 4436 * copy lmi_list and split it into components.
4417 4437 * then put the part of tail before $MACRO into path
4418 4438 * at pathend
4419 4439 */
4420 4440 diff = p - tail;
4421 4441 if (diff > 0)
4422 4442 bcopy(tail, pathend, diff);
4423 4443 path2 = pathend + diff;
4424 4444 p1 = libmacros[lmi].lmi_list;
4425 4445 while (p1 && (*p1 != '\0')) {
4426 4446 p2 = strchr(p1, ':');
4427 4447 if (p2) {
4428 4448 diff = p2 - p1;
4429 4449 bcopy(p1, path2, diff);
4430 4450 *(path2 + diff) = '\0';
4431 4451 } else {
4432 4452 diff = strlen(p1);
4433 4453 bcopy(p1, path2, diff + 1);
4434 4454 }
4435 4455 /* copy endp only if there isn't any more macro to expand */
4436 4456 if (!more_macro && (endp != NULL))
4437 4457 (void) strcat(path2, endp);
4438 4458 file = kobj_open_path(path, 1, 1);
4439 4459 if (file != (struct _buf *)-1) {
4440 4460 kobj_close_file(file);
4441 4461 /*
4442 4462 * if more macros to expand then call expand_libmacro(),
4443 4463 * else return path which has the whole path
4444 4464 */
4445 4465 if (!more_macro || (expand_libmacro(endp, path,
4446 4466 path2 + diff) != NULL)) {
4447 4467 return (path);
4448 4468 }
4449 4469 }
4450 4470 if (p2)
4451 4471 p1 = ++p2;
4452 4472 else
4453 4473 return (NULL);
4454 4474 }
4455 4475 return (NULL);
4456 4476 }
4457 4477
4458 4478 static void
4459 4479 tnf_add_notifyunload(kobj_notify_f *fp)
4460 4480 {
4461 4481 kobj_notify_list_t *entry;
4462 4482
4463 4483 entry = kobj_alloc(sizeof (kobj_notify_list_t), KM_WAIT);
4464 4484 entry->kn_type = KOBJ_NOTIFY_MODUNLOADING;
4465 4485 entry->kn_func = fp;
4466 4486 (void) kobj_notify_add(entry);
4467 4487 }
4468 4488
4469 4489 /* ARGSUSED */
4470 4490 static void
4471 4491 tnf_unsplice_probes(uint_t what, struct modctl *mod)
4472 4492 {
4473 4493 tnf_probe_control_t **p;
4474 4494 tnf_tag_data_t **q;
4475 4495 struct module *mp = mod->mod_mp;
4476 4496
4477 4497 if (!(mp->flags & KOBJ_TNF_PROBE))
4478 4498 return;
4479 4499
4480 4500 for (p = &__tnf_probe_list_head; *p; )
4481 4501 if (kobj_addrcheck(mp, (char *)*p) == 0)
4482 4502 *p = (*p)->next;
4483 4503 else
4484 4504 p = &(*p)->next;
4485 4505
4486 4506 for (q = &__tnf_tag_list_head; *q; )
4487 4507 if (kobj_addrcheck(mp, (char *)*q) == 0)
4488 4508 *q = (tnf_tag_data_t *)(*q)->tag_version;
4489 4509 else
4490 4510 q = (tnf_tag_data_t **)&(*q)->tag_version;
4491 4511
4492 4512 tnf_changed_probe_list = 1;
4493 4513 }
4494 4514
4495 4515 int
4496 4516 tnf_splice_probes(int boot_load, tnf_probe_control_t *plist,
4497 4517 tnf_tag_data_t *tlist)
4498 4518 {
4499 4519 int result = 0;
4500 4520 static int add_notify = 1;
4501 4521
4502 4522 if (plist) {
4503 4523 tnf_probe_control_t *pl;
4504 4524
4505 4525 for (pl = plist; pl->next; )
4506 4526 pl = pl->next;
4507 4527
4508 4528 if (!boot_load)
4509 4529 mutex_enter(&mod_lock);
4510 4530 tnf_changed_probe_list = 1;
4511 4531 pl->next = __tnf_probe_list_head;
4512 4532 __tnf_probe_list_head = plist;
4513 4533 if (!boot_load)
4514 4534 mutex_exit(&mod_lock);
4515 4535 result = 1;
4516 4536 }
4517 4537
4518 4538 if (tlist) {
4519 4539 tnf_tag_data_t *tl;
4520 4540
4521 4541 for (tl = tlist; tl->tag_version; )
4522 4542 tl = (tnf_tag_data_t *)tl->tag_version;
4523 4543
4524 4544 if (!boot_load)
4525 4545 mutex_enter(&mod_lock);
4526 4546 tl->tag_version = (tnf_tag_version_t *)__tnf_tag_list_head;
4527 4547 __tnf_tag_list_head = tlist;
4528 4548 if (!boot_load)
4529 4549 mutex_exit(&mod_lock);
4530 4550 result = 1;
4531 4551 }
4532 4552 if (!boot_load && result && add_notify) {
4533 4553 tnf_add_notifyunload(tnf_unsplice_probes);
4534 4554 add_notify = 0;
4535 4555 }
4536 4556 return (result);
4537 4557 }
4538 4558
4539 4559 char *kobj_file_buf;
4540 4560 int kobj_file_bufsize;
4541 4561
4542 4562 /*
4543 4563 * This code is for the purpose of manually recording which files
4544 4564 * needs to go into the boot archive on any given system.
4545 4565 *
4546 4566 * To enable the code, set kobj_file_bufsize in /etc/system
4547 4567 * and reboot the system, then use mdb to look at kobj_file_buf.
4548 4568 */
4549 4569 static void
4550 4570 kobj_record_file(char *filename)
4551 4571 {
4552 4572 static char *buf;
4553 4573 static int size = 0;
4554 4574 int n;
4555 4575
4556 4576 if (kobj_file_bufsize == 0) /* don't bother */
4557 4577 return;
4558 4578
4559 4579 if (kobj_file_buf == NULL) { /* allocate buffer */
4560 4580 size = kobj_file_bufsize;
4561 4581 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4562 4582 }
4563 4583
4564 4584 n = snprintf(buf, size, "%s\n", filename);
4565 4585 if (n > size)
4566 4586 n = size;
4567 4587 size -= n;
4568 4588 buf += n;
4569 4589 }
4570 4590
4571 4591 static int
4572 4592 kobj_boot_fstat(int fd, struct bootstat *stp)
4573 4593 {
4574 4594 #if defined(_OBP)
4575 4595 if (!standalone && _ioquiesced)
4576 4596 return (-1);
4577 4597 return (BOP_FSTAT(ops, fd, stp));
4578 4598 #else
4579 4599 return (BRD_FSTAT(bfs_ops, fd, stp));
4580 4600 #endif
4581 4601 }
4582 4602
4583 4603 static int
4584 4604 kobj_boot_open(char *filename, int flags)
4585 4605 {
4586 4606 #if defined(_OBP)
4587 4607
4588 4608 /*
4589 4609 * If io via bootops is quiesced, it means boot is no longer
4590 4610 * available to us. We make it look as if we can't open the
4591 4611 * named file - which is reasonably accurate.
4592 4612 */
4593 4613 if (!standalone && _ioquiesced)
4594 4614 return (-1);
4595 4615
4596 4616 kobj_record_file(filename);
4597 4617 return (BOP_OPEN(filename, flags));
4598 4618 #else /* x86 */
4599 4619 kobj_record_file(filename);
4600 4620 return (BRD_OPEN(bfs_ops, filename, flags));
4601 4621 #endif
4602 4622 }
4603 4623
4604 4624 static int
4605 4625 kobj_boot_close(int fd)
4606 4626 {
4607 4627 #if defined(_OBP)
4608 4628 if (!standalone && _ioquiesced)
4609 4629 return (-1);
4610 4630
4611 4631 return (BOP_CLOSE(fd));
4612 4632 #else /* x86 */
4613 4633 return (BRD_CLOSE(bfs_ops, fd));
4614 4634 #endif
4615 4635 }
4616 4636
4617 4637 /*ARGSUSED*/
4618 4638 static int
4619 4639 kobj_boot_seek(int fd, off_t hi, off_t lo)
4620 4640 {
4621 4641 #if defined(_OBP)
4622 4642 return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4623 4643 #else
4624 4644 return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4625 4645 #endif
4626 4646 }
4627 4647
4628 4648 static int
4629 4649 kobj_boot_read(int fd, caddr_t buf, size_t size)
4630 4650 {
4631 4651 #if defined(_OBP)
4632 4652 return (BOP_READ(fd, buf, size));
4633 4653 #else
4634 4654 return (BRD_READ(bfs_ops, fd, buf, size));
4635 4655 #endif
4636 4656 }
4637 4657
4638 4658 static int
4639 4659 kobj_boot_compinfo(int fd, struct compinfo *cb)
4640 4660 {
4641 4661 return (boot_compinfo(fd, cb));
4642 4662 }
4643 4663
4644 4664 /*
4645 4665 * Check if the file is compressed (for now we handle only gzip).
4646 4666 * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4647 4667 */
4648 4668 static int
4649 4669 kobj_is_compressed(intptr_t fd)
4650 4670 {
4651 4671 struct vnode *vp = (struct vnode *)fd;
4652 4672 ssize_t resid;
4653 4673 uint16_t magic_buf;
4654 4674 int err = 0;
4655 4675
4656 4676 if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4657 4677 sizeof (magic_buf), (offset_t)(0),
4658 4678 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
4659 4679
4660 4680 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4661 4681 "error code 0x%x\n", err);
4662 4682 return (0);
4663 4683 }
4664 4684
4665 4685 if (magic_buf == CH_MAGIC_GZIP)
4666 4686 return (CH_MAGIC_GZIP);
4667 4687
4668 4688 return (0);
4669 4689 }
|
↓ open down ↓ |
1621 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX