1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright 2014 Joyent, Inc. All rights reserved.
25 */
26
27 #include <sys/types.h>
28 #include <sys/sysmacros.h>
29 #include <sys/kmem.h>
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/errno.h>
33 #include <sys/mman.h>
34 #include <sys/cmn_err.h>
35 #include <sys/cred.h>
36 #include <sys/vmsystm.h>
37 #include <sys/machsystm.h>
38 #include <sys/debug.h>
39 #include <vm/as.h>
40 #include <vm/seg.h>
41 #include <sys/vmparam.h>
42 #include <sys/vfs.h>
43 #include <sys/elf.h>
44 #include <sys/machelf.h>
45 #include <sys/corectl.h>
46 #include <sys/exec.h>
47 #include <sys/exechdr.h>
48 #include <sys/autoconf.h>
49 #include <sys/mem.h>
50 #include <vm/seg_dev.h>
51 #include <sys/vmparam.h>
52 #include <sys/mmapobj.h>
53 #include <sys/atomic.h>
54
55 /*
56 * Theory statement:
57 *
58 * The main driving force behind mmapobj is to interpret and map ELF files
59 * inside of the kernel instead of having the linker be responsible for this.
60 *
61 * mmapobj also supports the AOUT 4.x binary format as well as flat files in
62 * a read only manner.
63 *
64 * When interpreting and mapping an ELF file, mmapobj will map each PT_LOAD
65 * or PT_SUNWBSS segment according to the ELF standard. Refer to the "Linker
66 * and Libraries Guide" for more information about the standard and mapping
67 * rules.
68 *
69 * Having mmapobj interpret and map objects will allow the kernel to make the
70 * best decision for where to place the mappings for said objects. Thus, we
71 * can make optimizations inside of the kernel for specific platforms or cache
72 * mapping information to make mapping objects faster. The cache is ignored
73 * if ASLR is enabled.
74 *
75 * The lib_va_hash will be one such optimization. For each ELF object that
76 * mmapobj is asked to interpret, we will attempt to cache the information
77 * about the PT_LOAD and PT_SUNWBSS sections to speed up future mappings of
78 * the same objects. We will cache up to LIBVA_CACHED_SEGS (see below) program
79 * headers which should cover a majority of the libraries out there without
80 * wasting space. In order to make sure that the cached information is valid,
81 * we check the passed in vnode's mtime and ctime to make sure the vnode
82 * has not been modified since the last time we used it.
83 *
84 * In addition, the lib_va_hash may contain a preferred starting VA for the
85 * object which can be useful for platforms which support a shared context.
86 * This will increase the likelyhood that library text can be shared among
87 * many different processes. We limit the reserved VA space for 32 bit objects
88 * in order to minimize fragmenting the processes address space.
89 *
90 * In addition to the above, the mmapobj interface allows for padding to be
91 * requested before the first mapping and after the last mapping created.
92 * When padding is requested, no additional optimizations will be made for
93 * that request.
94 */
95
96 /*
97 * Threshold to prevent allocating too much kernel memory to read in the
98 * program headers for an object. If it requires more than below,
99 * we will use a KM_NOSLEEP allocation to allocate memory to hold all of the
100 * program headers which could possibly fail. If less memory than below is
101 * needed, then we use a KM_SLEEP allocation and are willing to wait for the
102 * memory if we need to.
103 */
104 size_t mmapobj_alloc_threshold = 65536;
105
106 /* Debug stats for test coverage */
107 #ifdef DEBUG
108 struct mobj_stats {
109 uint_t mobjs_unmap_called;
110 uint_t mobjs_remap_devnull;
111 uint_t mobjs_lookup_start;
112 uint_t mobjs_alloc_start;
113 uint_t mobjs_alloc_vmem;
114 uint_t mobjs_add_collision;
115 uint_t mobjs_get_addr;
116 uint_t mobjs_map_flat_no_padding;
117 uint_t mobjs_map_flat_padding;
118 uint_t mobjs_map_ptload_text;
119 uint_t mobjs_map_ptload_initdata;
120 uint_t mobjs_map_ptload_preread;
121 uint_t mobjs_map_ptload_unaligned_text;
122 uint_t mobjs_map_ptload_unaligned_map_fail;
123 uint_t mobjs_map_ptload_unaligned_read_fail;
124 uint_t mobjs_zfoddiff;
125 uint_t mobjs_zfoddiff_nowrite;
126 uint_t mobjs_zfodextra;
127 uint_t mobjs_ptload_failed;
128 uint_t mobjs_map_elf_no_holes;
129 uint_t mobjs_unmap_hole;
130 uint_t mobjs_nomem_header;
131 uint_t mobjs_inval_header;
132 uint_t mobjs_overlap_header;
133 uint_t mobjs_np2_align;
134 uint_t mobjs_np2_align_overflow;
135 uint_t mobjs_exec_padding;
136 uint_t mobjs_exec_addr_mapped;
137 uint_t mobjs_exec_addr_devnull;
138 uint_t mobjs_exec_addr_in_use;
139 uint_t mobjs_lvp_found;
140 uint_t mobjs_no_loadable_yet;
141 uint_t mobjs_nothing_to_map;
142 uint_t mobjs_e2big;
143 uint_t mobjs_dyn_pad_align;
144 uint_t mobjs_dyn_pad_noalign;
145 uint_t mobjs_alloc_start_fail;
146 uint_t mobjs_lvp_nocache;
147 uint_t mobjs_extra_padding;
148 uint_t mobjs_lvp_not_needed;
149 uint_t mobjs_no_mem_map_sz;
150 uint_t mobjs_check_exec_failed;
151 uint_t mobjs_lvp_used;
152 uint_t mobjs_wrong_model;
153 uint_t mobjs_noexec_fs;
154 uint_t mobjs_e2big_et_rel;
155 uint_t mobjs_et_rel_mapped;
156 uint_t mobjs_unknown_elf_type;
157 uint_t mobjs_phent32_too_small;
158 uint_t mobjs_phent64_too_small;
159 uint_t mobjs_inval_elf_class;
160 uint_t mobjs_too_many_phdrs;
161 uint_t mobjs_no_phsize;
162 uint_t mobjs_phsize_large;
163 uint_t mobjs_phsize_xtralarge;
164 uint_t mobjs_fast_wrong_model;
165 uint_t mobjs_fast_e2big;
166 uint_t mobjs_fast;
167 uint_t mobjs_fast_success;
168 uint_t mobjs_fast_not_now;
169 uint_t mobjs_small_file;
170 uint_t mobjs_read_error;
171 uint_t mobjs_unsupported;
172 uint_t mobjs_flat_e2big;
173 uint_t mobjs_phent_align32;
174 uint_t mobjs_phent_align64;
175 uint_t mobjs_lib_va_find_hit;
176 uint_t mobjs_lib_va_find_delay_delete;
177 uint_t mobjs_lib_va_find_delete;
178 uint_t mobjs_lib_va_add_delay_delete;
179 uint_t mobjs_lib_va_add_delete;
180 uint_t mobjs_lib_va_create_failure;
181 uint_t mobjs_min_align;
182 #if defined(__sparc)
183 uint_t mobjs_aout_uzero_fault;
184 uint_t mobjs_aout_64bit_try;
185 uint_t mobjs_aout_noexec;
186 uint_t mobjs_aout_e2big;
187 uint_t mobjs_aout_lib;
188 uint_t mobjs_aout_fixed;
189 uint_t mobjs_aout_zfoddiff;
190 uint_t mobjs_aout_map_bss;
191 uint_t mobjs_aout_bss_fail;
192 uint_t mobjs_aout_nlist;
193 uint_t mobjs_aout_addr_in_use;
194 #endif
195 } mobj_stats;
196
197 #define MOBJ_STAT_ADD(stat) ((mobj_stats.mobjs_##stat)++)
198 #else
199 #define MOBJ_STAT_ADD(stat)
200 #endif
201
202 /*
203 * Check if addr is at or above the address space reserved for the stack.
204 * The stack is at the top of the address space for all sparc processes
205 * and 64 bit x86 processes. For 32 bit x86, the stack is not at the top
206 * of the address space and thus this check wil always return false for
207 * 32 bit x86 processes.
208 */
209 #if defined(__sparc)
210 #define OVERLAPS_STACK(addr, p) \
211 (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK)))
212 #elif defined(__amd64)
213 #define OVERLAPS_STACK(addr, p) \
214 ((p->p_model == DATAMODEL_LP64) && \
215 (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK))))
216 #elif defined(__i386)
217 #define OVERLAPS_STACK(addr, p) 0
218 #endif
219
220 /* lv_flags values - bitmap */
221 #define LV_ELF32 0x1 /* 32 bit ELF file */
222 #define LV_ELF64 0x2 /* 64 bit ELF file */
223 #define LV_DEL 0x4 /* delete when lv_refcnt hits zero */
224
225 /*
226 * Note: lv_num_segs will denote how many segments this file has and will
227 * only be set after the lv_mps array has been filled out.
228 * lv_mps can only be valid if lv_num_segs is non-zero.
229 */
230 struct lib_va {
231 struct lib_va *lv_next;
232 caddr_t lv_base_va; /* start va for library */
233 ssize_t lv_len; /* total va span of library */
234 size_t lv_align; /* minimum alignment */
235 uint64_t lv_nodeid; /* filesystem node id */
236 uint64_t lv_fsid; /* filesystem id */
237 timestruc_t lv_ctime; /* last time file was changed */
238 timestruc_t lv_mtime; /* or modified */
239 mmapobj_result_t lv_mps[LIBVA_CACHED_SEGS]; /* cached pheaders */
240 int lv_num_segs; /* # segs for this file */
241 int lv_flags;
242 uint_t lv_refcnt; /* number of holds on struct */
243 };
244
245 #define LIB_VA_SIZE 1024
246 #define LIB_VA_MASK (LIB_VA_SIZE - 1)
247 #define LIB_VA_MUTEX_SHIFT 3
248
249 #if (LIB_VA_SIZE & (LIB_VA_SIZE - 1))
250 #error "LIB_VA_SIZE is not a power of 2"
251 #endif
252
253 static struct lib_va *lib_va_hash[LIB_VA_SIZE];
254 static kmutex_t lib_va_hash_mutex[LIB_VA_SIZE >> LIB_VA_MUTEX_SHIFT];
255
256 #define LIB_VA_HASH_MUTEX(index) \
257 (&lib_va_hash_mutex[index >> LIB_VA_MUTEX_SHIFT])
258
259 #define LIB_VA_HASH(nodeid) \
260 (((nodeid) ^ ((nodeid) << 7) ^ ((nodeid) << 13)) & LIB_VA_MASK)
261
262 #define LIB_VA_MATCH_ID(arg1, arg2) \
263 ((arg1)->lv_nodeid == (arg2)->va_nodeid && \
264 (arg1)->lv_fsid == (arg2)->va_fsid)
265
266 #define LIB_VA_MATCH_TIME(arg1, arg2) \
267 ((arg1)->lv_ctime.tv_sec == (arg2)->va_ctime.tv_sec && \
268 (arg1)->lv_mtime.tv_sec == (arg2)->va_mtime.tv_sec && \
269 (arg1)->lv_ctime.tv_nsec == (arg2)->va_ctime.tv_nsec && \
270 (arg1)->lv_mtime.tv_nsec == (arg2)->va_mtime.tv_nsec)
271
272 #define LIB_VA_MATCH(arg1, arg2) \
273 (LIB_VA_MATCH_ID(arg1, arg2) && LIB_VA_MATCH_TIME(arg1, arg2))
274
275 /*
276 * lib_va will be used for optimized allocation of address ranges for
277 * libraries, such that subsequent mappings of the same library will attempt
278 * to use the same VA as previous mappings of that library.
279 * In order to map libraries at the same VA in many processes, we need to carve
280 * out our own address space for them which is unique across many processes.
281 * We use different arenas for 32 bit and 64 bit libraries.
282 *
283 * Since the 32 bit address space is relatively small, we limit the number of
284 * libraries which try to use consistent virtual addresses to lib_threshold.
285 * For 64 bit libraries there is no such limit since the address space is large.
286 */
287 static vmem_t *lib_va_32_arena;
288 static vmem_t *lib_va_64_arena;
289 uint_t lib_threshold = 20; /* modifiable via /etc/system */
290
291 static kmutex_t lib_va_init_mutex; /* no need to initialize */
292
293 /*
294 * Number of 32 bit and 64 bit libraries in lib_va hash.
295 */
296 static uint_t libs_mapped_32 = 0;
297 static uint_t libs_mapped_64 = 0;
298
299 /*
300 * Free up the resources associated with lvp as well as lvp itself.
301 * We also decrement the number of libraries mapped via a lib_va
302 * cached virtual address.
303 */
304 void
305 lib_va_free(struct lib_va *lvp)
306 {
307 int is_64bit = lvp->lv_flags & LV_ELF64;
308 ASSERT(lvp->lv_refcnt == 0);
309
310 if (lvp->lv_base_va != NULL) {
311 vmem_xfree(is_64bit ? lib_va_64_arena : lib_va_32_arena,
312 lvp->lv_base_va, lvp->lv_len);
313 if (is_64bit) {
314 atomic_dec_32(&libs_mapped_64);
315 } else {
316 atomic_dec_32(&libs_mapped_32);
317 }
318 }
319 kmem_free(lvp, sizeof (struct lib_va));
320 }
321
322 /*
323 * See if the file associated with the vap passed in is in the lib_va hash.
324 * If it is and the file has not been modified since last use, then
325 * return a pointer to that data. Otherwise, return NULL if the file has
326 * changed or the file was not found in the hash.
327 */
328 static struct lib_va *
329 lib_va_find(vattr_t *vap)
330 {
331 struct lib_va *lvp;
332 struct lib_va *del = NULL;
333 struct lib_va **tmp;
334 uint_t index;
335 index = LIB_VA_HASH(vap->va_nodeid);
336
337 mutex_enter(LIB_VA_HASH_MUTEX(index));
338 tmp = &lib_va_hash[index];
339 while (*tmp != NULL) {
340 lvp = *tmp;
341 if (LIB_VA_MATCH_ID(lvp, vap)) {
342 if (LIB_VA_MATCH_TIME(lvp, vap)) {
343 ASSERT((lvp->lv_flags & LV_DEL) == 0);
344 lvp->lv_refcnt++;
345 MOBJ_STAT_ADD(lib_va_find_hit);
346 } else {
347 /*
348 * file was updated since last use.
349 * need to remove it from list.
350 */
351 del = lvp;
352 *tmp = del->lv_next;
353 del->lv_next = NULL;
354 /*
355 * If we can't delete it now, mark it for later
356 */
357 if (del->lv_refcnt) {
358 MOBJ_STAT_ADD(lib_va_find_delay_delete);
359 del->lv_flags |= LV_DEL;
360 del = NULL;
361 }
362 lvp = NULL;
363 }
364 mutex_exit(LIB_VA_HASH_MUTEX(index));
365 if (del) {
366 ASSERT(del->lv_refcnt == 0);
367 MOBJ_STAT_ADD(lib_va_find_delete);
368 lib_va_free(del);
369 }
370 return (lvp);
371 }
372 tmp = &lvp->lv_next;
373 }
374 mutex_exit(LIB_VA_HASH_MUTEX(index));
375 return (NULL);
376 }
377
378 /*
379 * Add a new entry to the lib_va hash.
380 * Search the hash while holding the appropriate mutex to make sure that the
381 * data is not already in the cache. If we find data that is in the cache
382 * already and has not been modified since last use, we return NULL. If it
383 * has been modified since last use, we will remove that entry from
384 * the hash and it will be deleted once it's reference count reaches zero.
385 * If there is no current entry in the hash we will add the new entry and
386 * return it to the caller who is responsible for calling lib_va_release to
387 * drop their reference count on it.
388 *
389 * lv_num_segs will be set to zero since the caller needs to add that
390 * information to the data structure.
391 */
392 static struct lib_va *
393 lib_va_add_hash(caddr_t base_va, ssize_t len, size_t align, vattr_t *vap)
394 {
395 struct lib_va *lvp;
396 uint_t index;
397 model_t model;
398 struct lib_va **tmp;
399 struct lib_va *del = NULL;
400
401 model = get_udatamodel();
402 index = LIB_VA_HASH(vap->va_nodeid);
403
404 lvp = kmem_alloc(sizeof (struct lib_va), KM_SLEEP);
405
406 mutex_enter(LIB_VA_HASH_MUTEX(index));
407
408 /*
409 * Make sure not adding same data a second time.
410 * The hash chains should be relatively short and adding
411 * is a relatively rare event, so it's worth the check.
412 */
413 tmp = &lib_va_hash[index];
414 while (*tmp != NULL) {
415 if (LIB_VA_MATCH_ID(*tmp, vap)) {
416 if (LIB_VA_MATCH_TIME(*tmp, vap)) {
417 mutex_exit(LIB_VA_HASH_MUTEX(index));
418 kmem_free(lvp, sizeof (struct lib_va));
419 return (NULL);
420 }
421
422 /*
423 * We have the same nodeid and fsid but the file has
424 * been modified since we last saw it.
425 * Need to remove the old node and add this new
426 * one.
427 * Could probably use a callback mechanism to make
428 * this cleaner.
429 */
430 ASSERT(del == NULL);
431 del = *tmp;
432 *tmp = del->lv_next;
433 del->lv_next = NULL;
434
435 /*
436 * Check to see if we can free it. If lv_refcnt
437 * is greater than zero, than some other thread
438 * has a reference to the one we want to delete
439 * and we can not delete it. All of this is done
440 * under the lib_va_hash_mutex lock so it is atomic.
441 */
442 if (del->lv_refcnt) {
443 MOBJ_STAT_ADD(lib_va_add_delay_delete);
444 del->lv_flags |= LV_DEL;
445 del = NULL;
446 }
447 /* tmp is already advanced */
448 continue;
449 }
450 tmp = &((*tmp)->lv_next);
451 }
452
453 lvp->lv_base_va = base_va;
454 lvp->lv_len = len;
455 lvp->lv_align = align;
456 lvp->lv_nodeid = vap->va_nodeid;
457 lvp->lv_fsid = vap->va_fsid;
458 lvp->lv_ctime.tv_sec = vap->va_ctime.tv_sec;
459 lvp->lv_ctime.tv_nsec = vap->va_ctime.tv_nsec;
460 lvp->lv_mtime.tv_sec = vap->va_mtime.tv_sec;
461 lvp->lv_mtime.tv_nsec = vap->va_mtime.tv_nsec;
462 lvp->lv_next = NULL;
463 lvp->lv_refcnt = 1;
464
465 /* Caller responsible for filling this and lv_mps out */
466 lvp->lv_num_segs = 0;
467
468 if (model == DATAMODEL_LP64) {
469 lvp->lv_flags = LV_ELF64;
470 } else {
471 ASSERT(model == DATAMODEL_ILP32);
472 lvp->lv_flags = LV_ELF32;
473 }
474
475 if (base_va != NULL) {
476 if (model == DATAMODEL_LP64) {
477 atomic_inc_32(&libs_mapped_64);
478 } else {
479 ASSERT(model == DATAMODEL_ILP32);
480 atomic_inc_32(&libs_mapped_32);
481 }
482 }
483 ASSERT(*tmp == NULL);
484 *tmp = lvp;
485 mutex_exit(LIB_VA_HASH_MUTEX(index));
486 if (del) {
487 ASSERT(del->lv_refcnt == 0);
488 MOBJ_STAT_ADD(lib_va_add_delete);
489 lib_va_free(del);
490 }
491 return (lvp);
492 }
493
494 /*
495 * Release the hold on lvp which was acquired by lib_va_find or lib_va_add_hash.
496 * In addition, if this is the last hold and lvp is marked for deletion,
497 * free up it's reserved address space and free the structure.
498 */
499 static void
500 lib_va_release(struct lib_va *lvp)
501 {
502 uint_t index;
503 int to_del = 0;
504
505 ASSERT(lvp->lv_refcnt > 0);
506
507 index = LIB_VA_HASH(lvp->lv_nodeid);
508 mutex_enter(LIB_VA_HASH_MUTEX(index));
509 if (--lvp->lv_refcnt == 0 && (lvp->lv_flags & LV_DEL)) {
510 to_del = 1;
511 }
512 mutex_exit(LIB_VA_HASH_MUTEX(index));
513 if (to_del) {
514 ASSERT(lvp->lv_next == 0);
515 lib_va_free(lvp);
516 }
517 }
518
519 /*
520 * Dummy function for mapping through /dev/null
521 * Normally I would have used mmmmap in common/io/mem.c
522 * but that is a static function, and for /dev/null, it
523 * just returns -1.
524 */
525 /* ARGSUSED */
526 static int
527 mmapobj_dummy(dev_t dev, off_t off, int prot)
528 {
529 return (-1);
530 }
531
532 /*
533 * Called when an error occurred which requires mmapobj to return failure.
534 * All mapped objects will be unmapped and /dev/null mappings will be
535 * reclaimed if necessary.
536 * num_mapped is the number of elements of mrp which have been mapped, and
537 * num_segs is the total number of elements in mrp.
538 * For e_type ET_EXEC, we need to unmap all of the elements in mrp since
539 * we had already made reservations for them.
540 * If num_mapped equals num_segs, then we know that we had fully mapped
541 * the file and only need to clean up the segments described.
542 * If they are not equal, then for ET_DYN we will unmap the range from the
543 * end of the last mapped segment to the end of the last segment in mrp
544 * since we would have made a reservation for that memory earlier.
545 * If e_type is passed in as zero, num_mapped must equal num_segs.
546 */
547 void
548 mmapobj_unmap(mmapobj_result_t *mrp, int num_mapped, int num_segs,
549 ushort_t e_type)
550 {
551 int i;
552 struct as *as = curproc->p_as;
553 caddr_t addr;
554 size_t size;
555
556 if (e_type == ET_EXEC) {
557 num_mapped = num_segs;
558 }
559 #ifdef DEBUG
560 if (e_type == 0) {
561 ASSERT(num_mapped == num_segs);
562 }
563 #endif
564
565 MOBJ_STAT_ADD(unmap_called);
566 for (i = 0; i < num_mapped; i++) {
567
568 /*
569 * If we are going to have to create a mapping we need to
570 * make sure that no one else will use the address we
571 * need to remap between the time it is unmapped and
572 * mapped below.
573 */
574 if (mrp[i].mr_flags & MR_RESV) {
575 as_rangelock(as);
576 }
577 /* Always need to unmap what we mapped */
578 (void) as_unmap(as, mrp[i].mr_addr, mrp[i].mr_msize);
579
580 /* Need to reclaim /dev/null reservation from earlier */
581 if (mrp[i].mr_flags & MR_RESV) {
582 struct segdev_crargs dev_a;
583
584 ASSERT(e_type != ET_DYN);
585 /*
586 * Use seg_dev segment driver for /dev/null mapping.
587 */
588 dev_a.mapfunc = mmapobj_dummy;
589 dev_a.dev = makedevice(mm_major, M_NULL);
590 dev_a.offset = 0;
591 dev_a.type = 0; /* neither PRIVATE nor SHARED */
592 dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
593 dev_a.hat_attr = 0;
594 dev_a.hat_flags = 0;
595
596 (void) as_map(as, mrp[i].mr_addr, mrp[i].mr_msize,
597 segdev_create, &dev_a);
598 MOBJ_STAT_ADD(remap_devnull);
599 as_rangeunlock(as);
600 }
601 }
602
603 if (num_mapped != num_segs) {
604 ASSERT(e_type == ET_DYN);
605 /* Need to unmap any reservation made after last mapped seg */
606 if (num_mapped == 0) {
607 addr = mrp[0].mr_addr;
608 } else {
609 addr = mrp[num_mapped - 1].mr_addr +
610 mrp[num_mapped - 1].mr_msize;
611 }
612 size = (size_t)mrp[num_segs - 1].mr_addr +
613 mrp[num_segs - 1].mr_msize - (size_t)addr;
614 (void) as_unmap(as, addr, size);
615
616 /*
617 * Now we need to unmap the holes between mapped segs.
618 * Note that we have not mapped all of the segments and thus
619 * the holes between segments would not have been unmapped
620 * yet. If num_mapped == num_segs, then all of the holes
621 * between segments would have already been unmapped.
622 */
623
624 for (i = 1; i < num_mapped; i++) {
625 addr = mrp[i - 1].mr_addr + mrp[i - 1].mr_msize;
626 size = mrp[i].mr_addr - addr;
627 (void) as_unmap(as, addr, size);
628 }
629 }
630 }
631
632 /*
633 * We need to add the start address into mrp so that the unmap function
634 * has absolute addresses to use.
635 */
636 static void
637 mmapobj_unmap_exec(mmapobj_result_t *mrp, int num_mapped, caddr_t start_addr)
638 {
639 int i;
640
641 for (i = 0; i < num_mapped; i++) {
642 mrp[i].mr_addr += (size_t)start_addr;
643 }
644 mmapobj_unmap(mrp, num_mapped, num_mapped, ET_EXEC);
645 }
646
647 static caddr_t
648 mmapobj_lookup_start_addr(struct lib_va *lvp)
649 {
650 proc_t *p = curproc;
651 struct as *as = p->p_as;
652 struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
653 int error;
654 uint_t ma_flags = _MAP_LOW32;
655 caddr_t base = NULL;
656 size_t len;
657 size_t align;
658
659 ASSERT(lvp != NULL);
660 MOBJ_STAT_ADD(lookup_start);
661
662 as_rangelock(as);
663
664 base = lvp->lv_base_va;
665 len = lvp->lv_len;
666
667 /*
668 * If we don't have an expected base address, or the one that we want
669 * to use is not available or acceptable, go get an acceptable
670 * address range.
671 */
672 if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
673 valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
674 RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
675 if (lvp->lv_flags & LV_ELF64) {
676 ma_flags = 0;
677 }
678
679 align = lvp->lv_align;
680 if (align > 1) {
681 ma_flags |= MAP_ALIGN;
682 }
683
684 base = (caddr_t)align;
685 map_addr(&base, len, 0, 1, ma_flags);
686 }
687
688 /*
689 * Need to reserve the address space we're going to use.
690 * Don't reserve swap space since we'll be mapping over this.
691 */
692 if (base != NULL) {
693 crargs.flags |= MAP_NORESERVE;
694 error = as_map(as, base, len, segvn_create, &crargs);
695 if (error) {
696 base = NULL;
697 }
698 }
699
700 as_rangeunlock(as);
701 return (base);
702 }
703
704 /*
705 * Get the starting address for a given file to be mapped and return it
706 * to the caller. If we're using lib_va and we need to allocate an address,
707 * we will attempt to allocate it from the global reserved pool such that the
708 * same address can be used in the future for this file. If we can't use the
709 * reserved address then we just get one that will fit in our address space.
710 *
711 * Returns the starting virtual address for the range to be mapped or NULL
712 * if an error is encountered. If we successfully insert the requested info
713 * into the lib_va hash, then *lvpp will be set to point to this lib_va
714 * structure. The structure will have a hold on it and thus lib_va_release
715 * needs to be called on it by the caller. This function will not fill out
716 * lv_mps or lv_num_segs since it does not have enough information to do so.
717 * The caller is responsible for doing this making sure that any modifications
718 * to lv_mps are visible before setting lv_num_segs.
719 */
720 static caddr_t
721 mmapobj_alloc_start_addr(struct lib_va **lvpp, size_t len, int use_lib_va,
722 int randomize, size_t align, vattr_t *vap)
723 {
724 proc_t *p = curproc;
725 struct as *as = p->p_as;
726 struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
727 int error;
728 model_t model;
729 uint_t ma_flags = _MAP_LOW32;
730 caddr_t base = NULL;
731 vmem_t *model_vmem;
732 size_t lib_va_start;
733 size_t lib_va_end;
734 size_t lib_va_len;
735
736 ASSERT(lvpp != NULL);
737 ASSERT((randomize & use_lib_va) != 1);
738
739 MOBJ_STAT_ADD(alloc_start);
740 model = get_udatamodel();
741
742 if (model == DATAMODEL_LP64) {
743 ma_flags = 0;
744 model_vmem = lib_va_64_arena;
745 } else {
746 ASSERT(model == DATAMODEL_ILP32);
747 model_vmem = lib_va_32_arena;
748 }
749
750 if (align > 1) {
751 ma_flags |= MAP_ALIGN;
752 }
753
754 if (randomize != 0)
755 ma_flags |= _MAP_RANDOMIZE;
756
757 if (use_lib_va) {
758 /*
759 * The first time through, we need to setup the lib_va arenas.
760 * We call map_addr to find a suitable range of memory to map
761 * the given library, and we will set the highest address
762 * in our vmem arena to the end of this adddress range.
763 * We allow up to half of the address space to be used
764 * for lib_va addresses but we do not prevent any allocations
765 * in this range from other allocation paths.
766 */
767 if (lib_va_64_arena == NULL && model == DATAMODEL_LP64) {
768 mutex_enter(&lib_va_init_mutex);
769 if (lib_va_64_arena == NULL) {
770 base = (caddr_t)align;
771 as_rangelock(as);
772 map_addr(&base, len, 0, 1, ma_flags);
773 as_rangeunlock(as);
774 if (base == NULL) {
775 mutex_exit(&lib_va_init_mutex);
776 MOBJ_STAT_ADD(lib_va_create_failure);
777 goto nolibva;
778 }
779 lib_va_end = (size_t)base + len;
780 lib_va_len = lib_va_end >> 1;
781 lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
782 lib_va_start = lib_va_end - lib_va_len;
783
784 /*
785 * Need to make sure we avoid the address hole.
786 * We know lib_va_end is valid but we need to
787 * make sure lib_va_start is as well.
788 */
789 if ((lib_va_end > (size_t)hole_end) &&
790 (lib_va_start < (size_t)hole_end)) {
791 lib_va_start = P2ROUNDUP(
792 (size_t)hole_end, PAGESIZE);
793 lib_va_len = lib_va_end - lib_va_start;
794 }
795 lib_va_64_arena = vmem_create("lib_va_64",
796 (void *)lib_va_start, lib_va_len, PAGESIZE,
797 NULL, NULL, NULL, 0,
798 VM_NOSLEEP | VMC_IDENTIFIER);
799 if (lib_va_64_arena == NULL) {
800 mutex_exit(&lib_va_init_mutex);
801 goto nolibva;
802 }
803 }
804 model_vmem = lib_va_64_arena;
805 mutex_exit(&lib_va_init_mutex);
806 } else if (lib_va_32_arena == NULL &&
807 model == DATAMODEL_ILP32) {
808 mutex_enter(&lib_va_init_mutex);
809 if (lib_va_32_arena == NULL) {
810 base = (caddr_t)align;
811 as_rangelock(as);
812 map_addr(&base, len, 0, 1, ma_flags);
813 as_rangeunlock(as);
814 if (base == NULL) {
815 mutex_exit(&lib_va_init_mutex);
816 MOBJ_STAT_ADD(lib_va_create_failure);
817 goto nolibva;
818 }
819 lib_va_end = (size_t)base + len;
820 lib_va_len = lib_va_end >> 1;
821 lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
822 lib_va_start = lib_va_end - lib_va_len;
823 lib_va_32_arena = vmem_create("lib_va_32",
824 (void *)lib_va_start, lib_va_len, PAGESIZE,
825 NULL, NULL, NULL, 0,
826 VM_NOSLEEP | VMC_IDENTIFIER);
827 if (lib_va_32_arena == NULL) {
828 mutex_exit(&lib_va_init_mutex);
829 goto nolibva;
830 }
831 }
832 model_vmem = lib_va_32_arena;
833 mutex_exit(&lib_va_init_mutex);
834 }
835
836 if (model == DATAMODEL_LP64 || libs_mapped_32 < lib_threshold) {
837 base = vmem_xalloc(model_vmem, len, align, 0, 0, NULL,
838 NULL, VM_NOSLEEP | VM_ENDALLOC);
839 MOBJ_STAT_ADD(alloc_vmem);
840 }
841
842 /*
843 * Even if the address fails to fit in our address space,
844 * or we can't use a reserved address,
845 * we should still save it off in lib_va_hash.
846 */
847 *lvpp = lib_va_add_hash(base, len, align, vap);
848
849 /*
850 * Check for collision on insertion and free up our VA space.
851 * This is expected to be rare, so we'll just reset base to
852 * NULL instead of looking it up in the lib_va hash.
853 */
854 if (*lvpp == NULL) {
855 if (base != NULL) {
856 vmem_xfree(model_vmem, base, len);
857 base = NULL;
858 MOBJ_STAT_ADD(add_collision);
859 }
860 }
861 }
862
863 nolibva:
864 as_rangelock(as);
865
866 /*
867 * If we don't have an expected base address, or the one that we want
868 * to use is not available or acceptable, go get an acceptable
869 * address range.
870 *
871 * If ASLR is enabled, we should never have used the cache, and should
872 * also start our real work here, in the consequent of the next
873 * condition.
874 */
875 if (randomize != 0)
876 ASSERT(base == NULL);
877
878 if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
879 valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
880 RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
881 MOBJ_STAT_ADD(get_addr);
882 base = (caddr_t)align;
883 map_addr(&base, len, 0, 1, ma_flags);
884 }
885
886 /*
887 * Need to reserve the address space we're going to use.
888 * Don't reserve swap space since we'll be mapping over this.
889 */
890 if (base != NULL) {
891 /* Don't reserve swap space since we'll be mapping over this */
892 crargs.flags |= MAP_NORESERVE;
893 error = as_map(as, base, len, segvn_create, &crargs);
894 if (error) {
895 base = NULL;
896 }
897 }
898
899 as_rangeunlock(as);
900 return (base);
901 }
902
903 /*
904 * Map the file associated with vp into the address space as a single
905 * read only private mapping.
906 * Returns 0 for success, and non-zero for failure to map the file.
907 */
908 static int
909 mmapobj_map_flat(vnode_t *vp, mmapobj_result_t *mrp, size_t padding,
910 cred_t *fcred)
911 {
912 int error = 0;
913 struct as *as = curproc->p_as;
914 caddr_t addr = NULL;
915 caddr_t start_addr;
916 size_t len;
917 size_t pad_len;
918 int prot = PROT_USER | PROT_READ;
919 uint_t ma_flags = _MAP_LOW32;
920 vattr_t vattr;
921 struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
922
923 if (get_udatamodel() == DATAMODEL_LP64) {
924 ma_flags = 0;
925 }
926
927 vattr.va_mask = AT_SIZE;
928 error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
929 if (error) {
930 return (error);
931 }
932
933 len = vattr.va_size;
934
935 ma_flags |= MAP_PRIVATE;
936 if (padding == 0) {
937 MOBJ_STAT_ADD(map_flat_no_padding);
938 error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL,
939 ma_flags, fcred, NULL);
940 if (error == 0) {
941 mrp[0].mr_addr = addr;
942 mrp[0].mr_msize = len;
943 mrp[0].mr_fsize = len;
944 mrp[0].mr_offset = 0;
945 mrp[0].mr_prot = prot;
946 mrp[0].mr_flags = 0;
947 }
948 return (error);
949 }
950
951 /* padding was requested so there's more work to be done */
952 MOBJ_STAT_ADD(map_flat_padding);
953
954 /* No need to reserve swap space now since it will be reserved later */
955 crargs.flags |= MAP_NORESERVE;
956
957 /* Need to setup padding which can only be in PAGESIZE increments. */
958 ASSERT((padding & PAGEOFFSET) == 0);
959 pad_len = len + (2 * padding);
960
961 as_rangelock(as);
962 map_addr(&addr, pad_len, 0, 1, ma_flags);
963 error = as_map(as, addr, pad_len, segvn_create, &crargs);
964 as_rangeunlock(as);
965 if (error) {
966 return (error);
967 }
968 start_addr = addr;
969 addr += padding;
970 ma_flags |= MAP_FIXED;
971 error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL, ma_flags,
972 fcred, NULL);
973 if (error == 0) {
974 mrp[0].mr_addr = start_addr;
975 mrp[0].mr_msize = padding;
976 mrp[0].mr_fsize = 0;
977 mrp[0].mr_offset = 0;
978 mrp[0].mr_prot = 0;
979 mrp[0].mr_flags = MR_PADDING;
980
981 mrp[1].mr_addr = addr;
982 mrp[1].mr_msize = len;
983 mrp[1].mr_fsize = len;
984 mrp[1].mr_offset = 0;
985 mrp[1].mr_prot = prot;
986 mrp[1].mr_flags = 0;
987
988 mrp[2].mr_addr = addr + P2ROUNDUP(len, PAGESIZE);
989 mrp[2].mr_msize = padding;
990 mrp[2].mr_fsize = 0;
991 mrp[2].mr_offset = 0;
992 mrp[2].mr_prot = 0;
993 mrp[2].mr_flags = MR_PADDING;
994 } else {
995 /* Need to cleanup the as_map from earlier */
996 (void) as_unmap(as, start_addr, pad_len);
997 }
998 return (error);
999 }
1000
1001 /*
1002 * Map a PT_LOAD or PT_SUNWBSS section of an executable file into the user's
1003 * address space.
1004 * vp - vnode to be mapped in
1005 * addr - start address
1006 * len - length of vp to be mapped
1007 * zfodlen - length of zero filled memory after len above
1008 * offset - offset into file where mapping should start
1009 * prot - protections for this mapping
1010 * fcred - credentials for the file associated with vp at open time.
1011 */
1012 static int
1013 mmapobj_map_ptload(struct vnode *vp, caddr_t addr, size_t len, size_t zfodlen,
1014 off_t offset, int prot, cred_t *fcred)
1015 {
1016 int error = 0;
1017 caddr_t zfodbase, oldaddr;
1018 size_t oldlen;
1019 size_t end;
1020 size_t zfoddiff;
1021 label_t ljb;
1022 struct as *as = curproc->p_as;
1023 model_t model;
1024 int full_page;
1025
1026 /*
1027 * See if addr and offset are aligned such that we can map in
1028 * full pages instead of partial pages.
1029 */
1030 full_page = (((uintptr_t)addr & PAGEOFFSET) ==
1031 ((uintptr_t)offset & PAGEOFFSET));
1032
1033 model = get_udatamodel();
1034
1035 oldaddr = addr;
1036 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1037 if (len) {
1038 spgcnt_t availm, npages;
1039 int preread;
1040 uint_t mflag = MAP_PRIVATE | MAP_FIXED;
1041
1042 if (model == DATAMODEL_ILP32) {
1043 mflag |= _MAP_LOW32;
1044 }
1045 /* We may need to map in extra bytes */
1046 oldlen = len;
1047 len += ((size_t)oldaddr & PAGEOFFSET);
1048
1049 if (full_page) {
1050 offset = (off_t)((uintptr_t)offset & PAGEMASK);
1051 if ((prot & (PROT_WRITE | PROT_EXEC)) == PROT_EXEC) {
1052 mflag |= MAP_TEXT;
1053 MOBJ_STAT_ADD(map_ptload_text);
1054 } else {
1055 mflag |= MAP_INITDATA;
1056 MOBJ_STAT_ADD(map_ptload_initdata);
1057 }
1058
1059 /*
1060 * maxprot is passed as PROT_ALL so that mdb can
1061 * write to this segment.
1062 */
1063 if (error = VOP_MAP(vp, (offset_t)offset, as, &addr,
1064 len, prot, PROT_ALL, mflag, fcred, NULL)) {
1065 return (error);
1066 }
1067
1068 /*
1069 * If the segment can fit and is relatively small, then
1070 * we prefault the entire segment in. This is based
1071 * on the model that says the best working set of a
1072 * small program is all of its pages.
1073 * We only do this if freemem will not drop below
1074 * lotsfree since we don't want to induce paging.
1075 */
1076 npages = (spgcnt_t)btopr(len);
1077 availm = freemem - lotsfree;
1078 preread = (npages < availm && len < PGTHRESH) ? 1 : 0;
1079
1080 /*
1081 * If we aren't prefaulting the segment,
1082 * increment "deficit", if necessary to ensure
1083 * that pages will become available when this
1084 * process starts executing.
1085 */
1086 if (preread == 0 && npages > availm &&
1087 deficit < lotsfree) {
1088 deficit += MIN((pgcnt_t)(npages - availm),
1089 lotsfree - deficit);
1090 }
1091
1092 if (preread) {
1093 (void) as_faulta(as, addr, len);
1094 MOBJ_STAT_ADD(map_ptload_preread);
1095 }
1096 } else {
1097 /*
1098 * addr and offset were not aligned such that we could
1099 * use VOP_MAP, thus we need to as_map the memory we
1100 * need and then read the data in from disk.
1101 * This code path is a corner case which should never
1102 * be taken, but hand crafted binaries could trigger
1103 * this logic and it needs to work correctly.
1104 */
1105 MOBJ_STAT_ADD(map_ptload_unaligned_text);
1106 as_rangelock(as);
1107 (void) as_unmap(as, addr, len);
1108
1109 /*
1110 * We use zfod_argsp because we need to be able to
1111 * write to the mapping and then we'll change the
1112 * protections later if they are incorrect.
1113 */
1114 error = as_map(as, addr, len, segvn_create, zfod_argsp);
1115 as_rangeunlock(as);
1116 if (error) {
1117 MOBJ_STAT_ADD(map_ptload_unaligned_map_fail);
1118 return (error);
1119 }
1120
1121 /* Now read in the data from disk */
1122 error = vn_rdwr(UIO_READ, vp, oldaddr, oldlen, offset,
1123 UIO_USERSPACE, 0, (rlim64_t)0, fcred, NULL);
1124 if (error) {
1125 MOBJ_STAT_ADD(map_ptload_unaligned_read_fail);
1126 return (error);
1127 }
1128
1129 /*
1130 * Now set protections.
1131 */
1132 if (prot != PROT_ZFOD) {
1133 (void) as_setprot(as, addr, len, prot);
1134 }
1135 }
1136 }
1137
1138 if (zfodlen) {
1139 end = (size_t)addr + len;
1140 zfodbase = (caddr_t)P2ROUNDUP(end, PAGESIZE);
1141 zfoddiff = (uintptr_t)zfodbase - end;
1142 if (zfoddiff) {
1143 /*
1144 * Before we go to zero the remaining space on the last
1145 * page, make sure we have write permission.
1146 *
1147 * We need to be careful how we zero-fill the last page
1148 * if the protection does not include PROT_WRITE. Using
1149 * as_setprot() can cause the VM segment code to call
1150 * segvn_vpage(), which must allocate a page struct for
1151 * each page in the segment. If we have a very large
1152 * segment, this may fail, so we check for that, even
1153 * though we ignore other return values from as_setprot.
1154 */
1155 MOBJ_STAT_ADD(zfoddiff);
1156 if ((prot & PROT_WRITE) == 0) {
1157 if (as_setprot(as, (caddr_t)end, zfoddiff,
1158 prot | PROT_WRITE) == ENOMEM)
1159 return (ENOMEM);
1160 MOBJ_STAT_ADD(zfoddiff_nowrite);
1161 }
1162 if (on_fault(&ljb)) {
1163 no_fault();
1164 if ((prot & PROT_WRITE) == 0) {
1165 (void) as_setprot(as, (caddr_t)end,
1166 zfoddiff, prot);
1167 }
1168 return (EFAULT);
1169 }
1170 uzero((void *)end, zfoddiff);
1171 no_fault();
1172
1173 /*
1174 * Remove write protection to return to original state
1175 */
1176 if ((prot & PROT_WRITE) == 0) {
1177 (void) as_setprot(as, (caddr_t)end,
1178 zfoddiff, prot);
1179 }
1180 }
1181 if (zfodlen > zfoddiff) {
1182 struct segvn_crargs crargs =
1183 SEGVN_ZFOD_ARGS(prot, PROT_ALL);
1184
1185 MOBJ_STAT_ADD(zfodextra);
1186 zfodlen -= zfoddiff;
1187 crargs.szc = AS_MAP_NO_LPOOB;
1188
1189
1190 as_rangelock(as);
1191 (void) as_unmap(as, (caddr_t)zfodbase, zfodlen);
1192 error = as_map(as, (caddr_t)zfodbase,
1193 zfodlen, segvn_create, &crargs);
1194 as_rangeunlock(as);
1195 if (error) {
1196 return (error);
1197 }
1198 }
1199 }
1200 return (0);
1201 }
1202
1203 /*
1204 * Map the ELF file represented by vp into the users address space. The
1205 * first mapping will start at start_addr and there will be num_elements
1206 * mappings. The mappings are described by the data in mrp which may be
1207 * modified upon returning from this function.
1208 * Returns 0 for success or errno for failure.
1209 */
1210 static int
1211 mmapobj_map_elf(struct vnode *vp, caddr_t start_addr, mmapobj_result_t *mrp,
1212 int num_elements, cred_t *fcred, ushort_t e_type)
1213 {
1214 int i;
1215 int ret;
1216 caddr_t lo;
1217 caddr_t hi;
1218 struct as *as = curproc->p_as;
1219
1220 for (i = 0; i < num_elements; i++) {
1221 caddr_t addr;
1222 size_t p_memsz;
1223 size_t p_filesz;
1224 size_t zfodlen;
1225 offset_t p_offset;
1226 size_t dif;
1227 int prot;
1228
1229 /* Always need to adjust mr_addr */
1230 addr = start_addr + (size_t)(mrp[i].mr_addr);
1231 mrp[i].mr_addr =
1232 (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1233
1234 /* Padding has already been mapped */
1235 if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
1236 continue;
1237 }
1238 p_memsz = mrp[i].mr_msize;
1239 p_filesz = mrp[i].mr_fsize;
1240 zfodlen = p_memsz - p_filesz;
1241 p_offset = mrp[i].mr_offset;
1242 dif = (uintptr_t)(addr) & PAGEOFFSET;
1243 prot = mrp[i].mr_prot | PROT_USER;
1244 ret = mmapobj_map_ptload(vp, addr, p_filesz, zfodlen,
1245 p_offset, prot, fcred);
1246 if (ret != 0) {
1247 MOBJ_STAT_ADD(ptload_failed);
1248 mmapobj_unmap(mrp, i, num_elements, e_type);
1249 return (ret);
1250 }
1251
1252 /* Need to cleanup mrp to reflect the actual values used */
1253 mrp[i].mr_msize += dif;
1254 mrp[i].mr_offset = (size_t)addr & PAGEOFFSET;
1255 }
1256
1257 /* Also need to unmap any holes created above */
1258 if (num_elements == 1) {
1259 MOBJ_STAT_ADD(map_elf_no_holes);
1260 return (0);
1261 }
1262 if (e_type == ET_EXEC) {
1263 return (0);
1264 }
1265
1266 as_rangelock(as);
1267 lo = start_addr;
1268 hi = mrp[0].mr_addr;
1269
1270 /* Remove holes made by the rest of the segments */
1271 for (i = 0; i < num_elements - 1; i++) {
1272 lo = (caddr_t)P2ROUNDUP((size_t)(mrp[i].mr_addr) +
1273 mrp[i].mr_msize, PAGESIZE);
1274 hi = mrp[i + 1].mr_addr;
1275 if (lo < hi) {
1276 /*
1277 * If as_unmap fails we just use up a bit of extra
1278 * space
1279 */
1280 (void) as_unmap(as, (caddr_t)lo,
1281 (size_t)hi - (size_t)lo);
1282 MOBJ_STAT_ADD(unmap_hole);
1283 }
1284 }
1285 as_rangeunlock(as);
1286
1287 return (0);
1288 }
1289
1290 /* Ugly hack to get STRUCT_* macros to work below */
1291 struct myphdr {
1292 Phdr x; /* native version */
1293 };
1294
1295 struct myphdr32 {
1296 Elf32_Phdr x;
1297 };
1298
1299 /*
1300 * Calculate and return the number of loadable segments in the ELF Phdr
1301 * represented by phdrbase as well as the len of the total mapping and
1302 * the max alignment that is needed for a given segment. On success,
1303 * 0 is returned, and *len, *loadable and *align have been filled out.
1304 * On failure, errno will be returned, which in this case is ENOTSUP
1305 * if we were passed an ELF file with overlapping segments.
1306 */
1307 static int
1308 calc_loadable(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, size_t *len,
1309 int *loadable, size_t *align)
1310 {
1311 int i;
1312 int hsize;
1313 model_t model;
1314 ushort_t e_type = ehdrp->e_type; /* same offset 32 and 64 bit */
1315 uint_t p_type;
1316 offset_t p_offset;
1317 size_t p_memsz;
1318 size_t p_align;
1319 caddr_t vaddr;
1320 int num_segs = 0;
1321 caddr_t start_addr = NULL;
1322 caddr_t p_end = NULL;
1323 size_t max_align = 0;
1324 size_t min_align = PAGESIZE; /* needed for vmem_xalloc */
1325 STRUCT_HANDLE(myphdr, mph);
1326 #if defined(__sparc)
1327 extern int vac_size;
1328
1329 /*
1330 * Want to prevent aliasing by making the start address at least be
1331 * aligned to vac_size.
1332 */
1333 min_align = MAX(PAGESIZE, vac_size);
1334 #endif
1335
1336 model = get_udatamodel();
1337 STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1338
1339 /* hsize alignment should have been checked before calling this func */
1340 if (model == DATAMODEL_LP64) {
1341 hsize = ehdrp->e_phentsize;
1342 if (hsize & 7) {
1343 return (ENOTSUP);
1344 }
1345 } else {
1346 ASSERT(model == DATAMODEL_ILP32);
1347 hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
1348 if (hsize & 3) {
1349 return (ENOTSUP);
1350 }
1351 }
1352
1353 /*
1354 * Determine the span of all loadable segments and calculate the
1355 * number of loadable segments.
1356 */
1357 for (i = 0; i < nphdrs; i++) {
1358 p_type = STRUCT_FGET(mph, x.p_type);
1359 if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
1360 vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
1361 p_memsz = STRUCT_FGET(mph, x.p_memsz);
1362
1363 /*
1364 * Skip this header if it requests no memory to be
1365 * mapped.
1366 */
1367 if (p_memsz == 0) {
1368 STRUCT_SET_HANDLE(mph, model,
1369 (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1370 hsize));
1371 MOBJ_STAT_ADD(nomem_header);
1372 continue;
1373 }
1374 if (num_segs++ == 0) {
1375 /*
1376 * The p_vaddr of the first PT_LOAD segment
1377 * must either be NULL or within the first
1378 * page in order to be interpreted.
1379 * Otherwise, its an invalid file.
1380 */
1381 if (e_type == ET_DYN &&
1382 ((caddr_t)((uintptr_t)vaddr &
1383 (uintptr_t)PAGEMASK) != NULL)) {
1384 MOBJ_STAT_ADD(inval_header);
1385 return (ENOTSUP);
1386 }
1387 start_addr = vaddr;
1388 /*
1389 * For the first segment, we need to map from
1390 * the beginning of the file, so we will
1391 * adjust the size of the mapping to include
1392 * this memory.
1393 */
1394 p_offset = STRUCT_FGET(mph, x.p_offset);
1395 } else {
1396 p_offset = 0;
1397 }
1398 /*
1399 * Check to make sure that this mapping wouldn't
1400 * overlap a previous mapping.
1401 */
1402 if (vaddr < p_end) {
1403 MOBJ_STAT_ADD(overlap_header);
1404 return (ENOTSUP);
1405 }
1406
1407 p_end = vaddr + p_memsz + p_offset;
1408 p_end = (caddr_t)P2ROUNDUP((size_t)p_end, PAGESIZE);
1409
1410 p_align = STRUCT_FGET(mph, x.p_align);
1411 if (p_align > 1 && p_align > max_align) {
1412 max_align = p_align;
1413 if (max_align < min_align) {
1414 max_align = min_align;
1415 MOBJ_STAT_ADD(min_align);
1416 }
1417 }
1418 }
1419 STRUCT_SET_HANDLE(mph, model,
1420 (struct myphdr *)((size_t)STRUCT_BUF(mph) + hsize));
1421 }
1422
1423 /*
1424 * The alignment should be a power of 2, if it isn't we forgive it
1425 * and round up. On overflow, we'll set the alignment to max_align
1426 * rounded down to the nearest power of 2.
1427 */
1428 if (max_align > 0 && !ISP2(max_align)) {
1429 MOBJ_STAT_ADD(np2_align);
1430 *align = 2 * (1L << (highbit(max_align) - 1));
1431 if (*align < max_align ||
1432 (*align > UINT_MAX && model == DATAMODEL_ILP32)) {
1433 MOBJ_STAT_ADD(np2_align_overflow);
1434 *align = 1L << (highbit(max_align) - 1);
1435 }
1436 } else {
1437 *align = max_align;
1438 }
1439
1440 ASSERT(*align >= PAGESIZE || *align == 0);
1441
1442 *loadable = num_segs;
1443 *len = p_end - start_addr;
1444 return (0);
1445 }
1446
1447 /*
1448 * Check the address space to see if the virtual addresses to be used are
1449 * available. If they are not, return errno for failure. On success, 0
1450 * will be returned, and the virtual addresses for each mmapobj_result_t
1451 * will be reserved. Note that a reservation could have earlier been made
1452 * for a given segment via a /dev/null mapping. If that is the case, then
1453 * we can use that VA space for our mappings.
1454 * Note: this function will only be used for ET_EXEC binaries.
1455 */
1456 int
1457 check_exec_addrs(int loadable, mmapobj_result_t *mrp, caddr_t start_addr)
1458 {
1459 int i;
1460 struct as *as = curproc->p_as;
1461 struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
1462 int ret;
1463 caddr_t myaddr;
1464 size_t mylen;
1465 struct seg *seg;
1466
1467 /* No need to reserve swap space now since it will be reserved later */
1468 crargs.flags |= MAP_NORESERVE;
1469 as_rangelock(as);
1470 for (i = 0; i < loadable; i++) {
1471
1472 myaddr = start_addr + (size_t)mrp[i].mr_addr;
1473 mylen = mrp[i].mr_msize;
1474
1475 /* See if there is a hole in the as for this range */
1476 if (as_gap(as, mylen, &myaddr, &mylen, 0, NULL) == 0) {
1477 ASSERT(myaddr == start_addr + (size_t)mrp[i].mr_addr);
1478 ASSERT(mylen == mrp[i].mr_msize);
1479
1480 #ifdef DEBUG
1481 if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
1482 MOBJ_STAT_ADD(exec_padding);
1483 }
1484 #endif
1485 ret = as_map(as, myaddr, mylen, segvn_create, &crargs);
1486 if (ret) {
1487 as_rangeunlock(as);
1488 mmapobj_unmap_exec(mrp, i, start_addr);
1489 return (ret);
1490 }
1491 } else {
1492 /*
1493 * There is a mapping that exists in the range
1494 * so check to see if it was a "reservation"
1495 * from /dev/null. The mapping is from
1496 * /dev/null if the mapping comes from
1497 * segdev and the type is neither MAP_SHARED
1498 * nor MAP_PRIVATE.
1499 */
1500 AS_LOCK_ENTER(as, RW_READER);
1501 seg = as_findseg(as, myaddr, 0);
1502 MOBJ_STAT_ADD(exec_addr_mapped);
1503 if (seg && seg->s_ops == &segdev_ops &&
1504 ((SEGOP_GETTYPE(seg, myaddr) &
1505 (MAP_SHARED | MAP_PRIVATE)) == 0) &&
1506 myaddr >= seg->s_base &&
1507 myaddr + mylen <=
1508 seg->s_base + seg->s_size) {
1509 MOBJ_STAT_ADD(exec_addr_devnull);
1510 AS_LOCK_EXIT(as);
1511 (void) as_unmap(as, myaddr, mylen);
1512 ret = as_map(as, myaddr, mylen, segvn_create,
1513 &crargs);
1514 mrp[i].mr_flags |= MR_RESV;
1515 if (ret) {
1516 as_rangeunlock(as);
1517 /* Need to remap what we unmapped */
1518 mmapobj_unmap_exec(mrp, i + 1,
1519 start_addr);
1520 return (ret);
1521 }
1522 } else {
1523 AS_LOCK_EXIT(as);
1524 as_rangeunlock(as);
1525 mmapobj_unmap_exec(mrp, i, start_addr);
1526 MOBJ_STAT_ADD(exec_addr_in_use);
1527 return (EADDRINUSE);
1528 }
1529 }
1530 }
1531 as_rangeunlock(as);
1532 return (0);
1533 }
1534
1535 /*
1536 * Walk through the ELF program headers and extract all useful information
1537 * for PT_LOAD and PT_SUNWBSS segments into mrp.
1538 * Return 0 on success or error on failure.
1539 */
1540 static int
1541 process_phdrs(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, mmapobj_result_t *mrp,
1542 vnode_t *vp, uint_t *num_mapped, size_t padding, cred_t *fcred)
1543 {
1544 int i;
1545 caddr_t start_addr = NULL;
1546 caddr_t vaddr;
1547 size_t len = 0;
1548 size_t lib_len = 0;
1549 int ret;
1550 int prot;
1551 struct lib_va *lvp = NULL;
1552 vattr_t vattr;
1553 struct as *as = curproc->p_as;
1554 int error;
1555 int loadable = 0;
1556 int current = 0;
1557 int use_lib_va = 1;
1558 size_t align = 0;
1559 size_t add_pad = 0;
1560 int hdr_seen = 0;
1561 ushort_t e_type = ehdrp->e_type; /* same offset 32 and 64 bit */
1562 uint_t p_type;
1563 offset_t p_offset;
1564 size_t p_memsz;
1565 size_t p_filesz;
1566 uint_t p_flags;
1567 int hsize;
1568 model_t model;
1569 STRUCT_HANDLE(myphdr, mph);
1570
1571 model = get_udatamodel();
1572 STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1573
1574 /*
1575 * Need to make sure that hsize is aligned properly.
1576 * For 32bit processes, 4 byte alignment is required.
1577 * For 64bit processes, 8 byte alignment is required.
1578 * If the alignment isn't correct, we need to return failure
1579 * since it could cause an alignment error panic while walking
1580 * the phdr array.
1581 */
1582 if (model == DATAMODEL_LP64) {
1583 hsize = ehdrp->e_phentsize;
1584 if (hsize & 7) {
1585 MOBJ_STAT_ADD(phent_align64);
1586 return (ENOTSUP);
1587 }
1588 } else {
1589 ASSERT(model == DATAMODEL_ILP32);
1590 hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
1591 if (hsize & 3) {
1592 MOBJ_STAT_ADD(phent_align32);
1593 return (ENOTSUP);
1594 }
1595 }
1596
1597 if ((padding != 0) || secflag_enabled(curproc, PROC_SEC_ASLR)) {
1598 use_lib_va = 0;
1599 }
1600 if (e_type == ET_DYN) {
1601 vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME;
1602 error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
1603 if (error) {
1604 return (error);
1605 }
1606 /* Check to see if we already have a description for this lib */
1607 if (!secflag_enabled(curproc, PROC_SEC_ASLR))
1608 lvp = lib_va_find(&vattr);
1609
1610 if (lvp != NULL) {
1611 MOBJ_STAT_ADD(lvp_found);
1612 if (use_lib_va) {
1613 start_addr = mmapobj_lookup_start_addr(lvp);
1614 if (start_addr == NULL) {
1615 lib_va_release(lvp);
1616 return (ENOMEM);
1617 }
1618 }
1619
1620 /*
1621 * loadable may be zero if the original allocator
1622 * of lvp hasn't finished setting it up but the rest
1623 * of the fields will be accurate.
1624 */
1625 loadable = lvp->lv_num_segs;
1626 len = lvp->lv_len;
1627 align = lvp->lv_align;
1628 }
1629 }
1630
1631 /*
1632 * Determine the span of all loadable segments and calculate the
1633 * number of loadable segments, the total len spanned by the mappings
1634 * and the max alignment, if we didn't get them above.
1635 */
1636 if (loadable == 0) {
1637 MOBJ_STAT_ADD(no_loadable_yet);
1638 ret = calc_loadable(ehdrp, phdrbase, nphdrs, &len,
1639 &loadable, &align);
1640 if (ret != 0) {
1641 /*
1642 * Since it'd be an invalid file, we shouldn't have
1643 * cached it previously.
1644 */
1645 ASSERT(lvp == NULL);
1646 return (ret);
1647 }
1648 #ifdef DEBUG
1649 if (lvp) {
1650 ASSERT(len == lvp->lv_len);
1651 ASSERT(align == lvp->lv_align);
1652 }
1653 #endif
1654 }
1655
1656 /* Make sure there's something to map. */
1657 if (len == 0 || loadable == 0) {
1658 /*
1659 * Since it'd be an invalid file, we shouldn't have
1660 * cached it previously.
1661 */
1662 ASSERT(lvp == NULL);
1663 MOBJ_STAT_ADD(nothing_to_map);
1664 return (ENOTSUP);
1665 }
1666
1667 lib_len = len;
1668 if (padding != 0) {
1669 loadable += 2;
1670 }
1671 if (loadable > *num_mapped) {
1672 *num_mapped = loadable;
1673 /* cleanup previous reservation */
1674 if (start_addr) {
1675 (void) as_unmap(as, start_addr, lib_len);
1676 }
1677 MOBJ_STAT_ADD(e2big);
1678 if (lvp) {
1679 lib_va_release(lvp);
1680 }
1681 return (E2BIG);
1682 }
1683
1684 /*
1685 * We now know the size of the object to map and now we need to
1686 * get the start address to map it at. It's possible we already
1687 * have it if we found all the info we need in the lib_va cache.
1688 */
1689 if (e_type == ET_DYN && start_addr == NULL) {
1690 /*
1691 * Need to make sure padding does not throw off
1692 * required alignment. We can only specify an
1693 * alignment for the starting address to be mapped,
1694 * so we round padding up to the alignment and map
1695 * from there and then throw out the extra later.
1696 */
1697 if (padding != 0) {
1698 if (align > 1) {
1699 add_pad = P2ROUNDUP(padding, align);
1700 len += add_pad;
1701 MOBJ_STAT_ADD(dyn_pad_align);
1702 } else {
1703 MOBJ_STAT_ADD(dyn_pad_noalign);
1704 len += padding; /* at beginning */
1705 }
1706 len += padding; /* at end of mapping */
1707 }
1708 /*
1709 * At this point, if lvp is non-NULL, then above we
1710 * already found it in the cache but did not get
1711 * the start address since we were not going to use lib_va.
1712 * Since we know that lib_va will not be used, it's safe
1713 * to call mmapobj_alloc_start_addr and know that lvp
1714 * will not be modified.
1715 */
1716 ASSERT(lvp ? use_lib_va == 0 : 1);
1717 start_addr = mmapobj_alloc_start_addr(&lvp, len,
1718 use_lib_va,
1719 secflag_enabled(curproc, PROC_SEC_ASLR),
1720 align, &vattr);
1721 if (start_addr == NULL) {
1722 if (lvp) {
1723 lib_va_release(lvp);
1724 }
1725 MOBJ_STAT_ADD(alloc_start_fail);
1726 return (ENOMEM);
1727 }
1728 /*
1729 * If we can't cache it, no need to hang on to it.
1730 * Setting lv_num_segs to non-zero will make that
1731 * field active and since there are too many segments
1732 * to cache, all future users will not try to use lv_mps.
1733 */
1734 if (lvp != NULL && loadable > LIBVA_CACHED_SEGS && use_lib_va) {
1735 lvp->lv_num_segs = loadable;
1736 lib_va_release(lvp);
1737 lvp = NULL;
1738 MOBJ_STAT_ADD(lvp_nocache);
1739 }
1740 /*
1741 * Free the beginning of the mapping if the padding
1742 * was not aligned correctly.
1743 */
1744 if (padding != 0 && add_pad != padding) {
1745 (void) as_unmap(as, start_addr,
1746 add_pad - padding);
1747 start_addr += (add_pad - padding);
1748 MOBJ_STAT_ADD(extra_padding);
1749 }
1750 }
1751
1752 /*
1753 * At this point, we have reserved the virtual address space
1754 * for our mappings. Now we need to start filling out the mrp
1755 * array to describe all of the individual mappings we are going
1756 * to return.
1757 * For ET_EXEC there has been no memory reservation since we are
1758 * using fixed addresses. While filling in the mrp array below,
1759 * we will have the first segment biased to start at addr 0
1760 * and the rest will be biased by this same amount. Thus if there
1761 * is padding, the first padding will start at addr 0, and the next
1762 * segment will start at the value of padding.
1763 */
1764
1765 /* We'll fill out padding later, so start filling in mrp at index 1 */
1766 if (padding != 0) {
1767 current = 1;
1768 }
1769
1770 /* If we have no more need for lvp let it go now */
1771 if (lvp != NULL && use_lib_va == 0) {
1772 lib_va_release(lvp);
1773 MOBJ_STAT_ADD(lvp_not_needed);
1774 lvp = NULL;
1775 }
1776
1777 /* Now fill out the mrp structs from the program headers */
1778 STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1779 for (i = 0; i < nphdrs; i++) {
1780 p_type = STRUCT_FGET(mph, x.p_type);
1781 if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
1782 vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
1783 p_memsz = STRUCT_FGET(mph, x.p_memsz);
1784 p_filesz = STRUCT_FGET(mph, x.p_filesz);
1785 p_offset = STRUCT_FGET(mph, x.p_offset);
1786 p_flags = STRUCT_FGET(mph, x.p_flags);
1787
1788 /*
1789 * Skip this header if it requests no memory to be
1790 * mapped.
1791 */
1792 if (p_memsz == 0) {
1793 STRUCT_SET_HANDLE(mph, model,
1794 (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1795 hsize));
1796 MOBJ_STAT_ADD(no_mem_map_sz);
1797 continue;
1798 }
1799
1800 prot = 0;
1801 if (p_flags & PF_R)
1802 prot |= PROT_READ;
1803 if (p_flags & PF_W)
1804 prot |= PROT_WRITE;
1805 if (p_flags & PF_X)
1806 prot |= PROT_EXEC;
1807
1808 ASSERT(current < loadable);
1809 mrp[current].mr_msize = p_memsz;
1810 mrp[current].mr_fsize = p_filesz;
1811 mrp[current].mr_offset = p_offset;
1812 mrp[current].mr_prot = prot;
1813
1814 if (hdr_seen == 0 && p_filesz != 0) {
1815 mrp[current].mr_flags = MR_HDR_ELF;
1816 /*
1817 * We modify mr_offset because we
1818 * need to map the ELF header as well, and if
1819 * we didn't then the header could be left out
1820 * of the mapping that we will create later.
1821 * Since we're removing the offset, we need to
1822 * account for that in the other fields as well
1823 * since we will be mapping the memory from 0
1824 * to p_offset.
1825 */
1826 if (e_type == ET_DYN) {
1827 mrp[current].mr_offset = 0;
1828 mrp[current].mr_msize += p_offset;
1829 mrp[current].mr_fsize += p_offset;
1830 } else {
1831 ASSERT(e_type == ET_EXEC);
1832 /*
1833 * Save off the start addr which will be
1834 * our bias for the rest of the
1835 * ET_EXEC mappings.
1836 */
1837 start_addr = vaddr - padding;
1838 }
1839 mrp[current].mr_addr = (caddr_t)padding;
1840 hdr_seen = 1;
1841 } else {
1842 if (e_type == ET_EXEC) {
1843 /* bias mr_addr */
1844 mrp[current].mr_addr =
1845 vaddr - (size_t)start_addr;
1846 } else {
1847 mrp[current].mr_addr = vaddr + padding;
1848 }
1849 mrp[current].mr_flags = 0;
1850 }
1851 current++;
1852 }
1853
1854 /* Move to next phdr */
1855 STRUCT_SET_HANDLE(mph, model,
1856 (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1857 hsize));
1858 }
1859
1860 /* Now fill out the padding segments */
1861 if (padding != 0) {
1862 mrp[0].mr_addr = NULL;
1863 mrp[0].mr_msize = padding;
1864 mrp[0].mr_fsize = 0;
1865 mrp[0].mr_offset = 0;
1866 mrp[0].mr_prot = 0;
1867 mrp[0].mr_flags = MR_PADDING;
1868
1869 /* Setup padding for the last segment */
1870 ASSERT(current == loadable - 1);
1871 mrp[current].mr_addr = (caddr_t)lib_len + padding;
1872 mrp[current].mr_msize = padding;
1873 mrp[current].mr_fsize = 0;
1874 mrp[current].mr_offset = 0;
1875 mrp[current].mr_prot = 0;
1876 mrp[current].mr_flags = MR_PADDING;
1877 }
1878
1879 /*
1880 * Need to make sure address ranges desired are not in use or
1881 * are previously allocated reservations from /dev/null. For
1882 * ET_DYN, we already made sure our address range was free.
1883 */
1884 if (e_type == ET_EXEC) {
1885 ret = check_exec_addrs(loadable, mrp, start_addr);
1886 if (ret != 0) {
1887 ASSERT(lvp == NULL);
1888 MOBJ_STAT_ADD(check_exec_failed);
1889 return (ret);
1890 }
1891 }
1892
1893 /* Finish up our business with lvp. */
1894 if (lvp) {
1895 ASSERT(e_type == ET_DYN);
1896 if (lvp->lv_num_segs == 0 && loadable <= LIBVA_CACHED_SEGS) {
1897 bcopy(mrp, lvp->lv_mps,
1898 loadable * sizeof (mmapobj_result_t));
1899 membar_producer();
1900 }
1901 /*
1902 * Setting lv_num_segs to a non-zero value indicates that
1903 * lv_mps is now valid and can be used by other threads.
1904 * So, the above stores need to finish before lv_num_segs
1905 * is updated. lv_mps is only valid if lv_num_segs is
1906 * greater than LIBVA_CACHED_SEGS.
1907 */
1908 lvp->lv_num_segs = loadable;
1909 lib_va_release(lvp);
1910 MOBJ_STAT_ADD(lvp_used);
1911 }
1912
1913 /* Now that we have mrp completely filled out go map it */
1914 ret = mmapobj_map_elf(vp, start_addr, mrp, loadable, fcred, e_type);
1915 if (ret == 0) {
1916 *num_mapped = loadable;
1917 }
1918
1919 return (ret);
1920 }
1921
1922 /*
1923 * Take the ELF file passed in, and do the work of mapping it.
1924 * num_mapped in - # elements in user buffer
1925 * num_mapped out - # sections mapped and length of mrp array if
1926 * no errors.
1927 */
1928 static int
1929 doelfwork(Ehdr *ehdrp, vnode_t *vp, mmapobj_result_t *mrp,
1930 uint_t *num_mapped, size_t padding, cred_t *fcred)
1931 {
1932 int error;
1933 offset_t phoff;
1934 int nphdrs;
1935 unsigned char ei_class;
1936 unsigned short phentsize;
1937 ssize_t phsizep;
1938 caddr_t phbasep;
1939 int to_map;
1940 model_t model;
1941
1942 ei_class = ehdrp->e_ident[EI_CLASS];
1943 model = get_udatamodel();
1944 if ((model == DATAMODEL_ILP32 && ei_class == ELFCLASS64) ||
1945 (model == DATAMODEL_LP64 && ei_class == ELFCLASS32)) {
1946 MOBJ_STAT_ADD(wrong_model);
1947 return (ENOTSUP);
1948 }
1949
1950 /* Can't execute code from "noexec" mounted filesystem. */
1951 if (ehdrp->e_type == ET_EXEC &&
1952 (vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) {
1953 MOBJ_STAT_ADD(noexec_fs);
1954 return (EACCES);
1955 }
1956
1957 /*
1958 * Relocatable and core files are mapped as a single flat file
1959 * since no interpretation is done on them by mmapobj.
1960 */
1961 if (ehdrp->e_type == ET_REL || ehdrp->e_type == ET_CORE) {
1962 to_map = padding ? 3 : 1;
1963 if (*num_mapped < to_map) {
1964 *num_mapped = to_map;
1965 MOBJ_STAT_ADD(e2big_et_rel);
1966 return (E2BIG);
1967 }
1968 error = mmapobj_map_flat(vp, mrp, padding, fcred);
1969 if (error == 0) {
1970 *num_mapped = to_map;
1971 mrp[padding ? 1 : 0].mr_flags = MR_HDR_ELF;
1972 MOBJ_STAT_ADD(et_rel_mapped);
1973 }
1974 return (error);
1975 }
1976
1977 /* Check for an unknown ELF type */
1978 if (ehdrp->e_type != ET_EXEC && ehdrp->e_type != ET_DYN) {
1979 MOBJ_STAT_ADD(unknown_elf_type);
1980 return (ENOTSUP);
1981 }
1982
1983 if (ei_class == ELFCLASS32) {
1984 Elf32_Ehdr *e32hdr = (Elf32_Ehdr *)ehdrp;
1985 ASSERT(model == DATAMODEL_ILP32);
1986 nphdrs = e32hdr->e_phnum;
1987 phentsize = e32hdr->e_phentsize;
1988 if (phentsize < sizeof (Elf32_Phdr)) {
1989 MOBJ_STAT_ADD(phent32_too_small);
1990 return (ENOTSUP);
1991 }
1992 phoff = e32hdr->e_phoff;
1993 } else if (ei_class == ELFCLASS64) {
1994 Elf64_Ehdr *e64hdr = (Elf64_Ehdr *)ehdrp;
1995 ASSERT(model == DATAMODEL_LP64);
1996 nphdrs = e64hdr->e_phnum;
1997 phentsize = e64hdr->e_phentsize;
1998 if (phentsize < sizeof (Elf64_Phdr)) {
1999 MOBJ_STAT_ADD(phent64_too_small);
2000 return (ENOTSUP);
2001 }
2002 phoff = e64hdr->e_phoff;
2003 } else {
2004 /* fallthrough case for an invalid ELF class */
2005 MOBJ_STAT_ADD(inval_elf_class);
2006 return (ENOTSUP);
2007 }
2008
2009 /*
2010 * nphdrs should only have this value for core files which are handled
2011 * above as a single mapping. If other file types ever use this
2012 * sentinel, then we'll add the support needed to handle this here.
2013 */
2014 if (nphdrs == PN_XNUM) {
2015 MOBJ_STAT_ADD(too_many_phdrs);
2016 return (ENOTSUP);
2017 }
2018
2019 phsizep = nphdrs * phentsize;
2020
2021 if (phsizep == 0) {
2022 MOBJ_STAT_ADD(no_phsize);
2023 return (ENOTSUP);
2024 }
2025
2026 /* Make sure we only wait for memory if it's a reasonable request */
2027 if (phsizep > mmapobj_alloc_threshold) {
2028 MOBJ_STAT_ADD(phsize_large);
2029 if ((phbasep = kmem_alloc(phsizep, KM_NOSLEEP)) == NULL) {
2030 MOBJ_STAT_ADD(phsize_xtralarge);
2031 return (ENOMEM);
2032 }
2033 } else {
2034 phbasep = kmem_alloc(phsizep, KM_SLEEP);
2035 }
2036
2037 if ((error = vn_rdwr(UIO_READ, vp, phbasep, phsizep,
2038 (offset_t)phoff, UIO_SYSSPACE, 0, (rlim64_t)0,
2039 fcred, NULL)) != 0) {
2040 kmem_free(phbasep, phsizep);
2041 return (error);
2042 }
2043
2044 /* Now process the phdr's */
2045 error = process_phdrs(ehdrp, phbasep, nphdrs, mrp, vp, num_mapped,
2046 padding, fcred);
2047 kmem_free(phbasep, phsizep);
2048 return (error);
2049 }
2050
2051 #if defined(__sparc)
2052 /*
2053 * Hack to support 64 bit kernels running AOUT 4.x programs.
2054 * This is the sizeof (struct nlist) for a 32 bit kernel.
2055 * Since AOUT programs are 32 bit only, they will never use the 64 bit
2056 * sizeof (struct nlist) and thus creating a #define is the simplest
2057 * way around this since this is a format which is not being updated.
2058 * This will be used in the place of sizeof (struct nlist) below.
2059 */
2060 #define NLIST_SIZE (0xC)
2061
2062 static int
2063 doaoutwork(vnode_t *vp, mmapobj_result_t *mrp,
2064 uint_t *num_mapped, struct exec *hdr, cred_t *fcred)
2065 {
2066 int error;
2067 size_t size;
2068 size_t osize;
2069 size_t nsize; /* nlist size */
2070 size_t msize;
2071 size_t zfoddiff;
2072 caddr_t addr;
2073 caddr_t start_addr;
2074 struct as *as = curproc->p_as;
2075 int prot = PROT_USER | PROT_READ | PROT_EXEC;
2076 uint_t mflag = MAP_PRIVATE | _MAP_LOW32;
2077 offset_t off = 0;
2078 int segnum = 0;
2079 uint_t to_map;
2080 int is_library = 0;
2081 struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
2082
2083 /* Only 32bit apps supported by this file format */
2084 if (get_udatamodel() != DATAMODEL_ILP32) {
2085 MOBJ_STAT_ADD(aout_64bit_try);
2086 return (ENOTSUP);
2087 }
2088
2089 /* Check to see if this is a library */
2090 if (hdr->a_magic == ZMAGIC && hdr->a_entry < PAGESIZE) {
2091 is_library = 1;
2092 }
2093
2094 /* Can't execute code from "noexec" mounted filesystem. */
2095 if (((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) && (is_library == 0)) {
2096 MOBJ_STAT_ADD(aout_noexec);
2097 return (EACCES);
2098 }
2099
2100 /*
2101 * There are 2 ways to calculate the mapped size of executable:
2102 * 1) rounded text size + data size + bss size.
2103 * 2) starting offset for text + text size + data size + text relocation
2104 * size + data relocation size + room for nlist data structure.
2105 *
2106 * The larger of the two sizes will be used to map this binary.
2107 */
2108 osize = P2ROUNDUP(hdr->a_text, PAGESIZE) + hdr->a_data + hdr->a_bss;
2109
2110 off = hdr->a_magic == ZMAGIC ? 0 : sizeof (struct exec);
2111
2112 nsize = off + hdr->a_text + hdr->a_data + hdr->a_trsize +
2113 hdr->a_drsize + NLIST_SIZE;
2114
2115 size = MAX(osize, nsize);
2116 if (size != nsize) {
2117 nsize = 0;
2118 }
2119
2120 /*
2121 * 1 seg for text and 1 seg for initialized data.
2122 * 1 seg for bss (if can't fit in leftover space of init data)
2123 * 1 seg for nlist if needed.
2124 */
2125 to_map = 2 + (nsize ? 1 : 0) +
2126 (hdr->a_bss > PAGESIZE - P2PHASE(hdr->a_data, PAGESIZE) ? 1 : 0);
2127 if (*num_mapped < to_map) {
2128 *num_mapped = to_map;
2129 MOBJ_STAT_ADD(aout_e2big);
2130 return (E2BIG);
2131 }
2132
2133 /* Reserve address space for the whole mapping */
2134 if (is_library) {
2135 /* We'll let VOP_MAP below pick our address for us */
2136 addr = NULL;
2137 MOBJ_STAT_ADD(aout_lib);
2138 } else {
2139 /*
2140 * default start address for fixed binaries from AOUT 4.x
2141 * standard.
2142 */
2143 MOBJ_STAT_ADD(aout_fixed);
2144 mflag |= MAP_FIXED;
2145 addr = (caddr_t)0x2000;
2146 as_rangelock(as);
2147 if (as_gap(as, size, &addr, &size, 0, NULL) != 0) {
2148 as_rangeunlock(as);
2149 MOBJ_STAT_ADD(aout_addr_in_use);
2150 return (EADDRINUSE);
2151 }
2152 crargs.flags |= MAP_NORESERVE;
2153 error = as_map(as, addr, size, segvn_create, &crargs);
2154 ASSERT(addr == (caddr_t)0x2000);
2155 as_rangeunlock(as);
2156 }
2157
2158 start_addr = addr;
2159 osize = size;
2160
2161 /*
2162 * Map as large as we need, backed by file, this will be text, and
2163 * possibly the nlist segment. We map over this mapping for bss and
2164 * initialized data segments.
2165 */
2166 error = VOP_MAP(vp, off, as, &addr, size, prot, PROT_ALL,
2167 mflag, fcred, NULL);
2168 if (error) {
2169 if (!is_library) {
2170 (void) as_unmap(as, start_addr, osize);
2171 }
2172 return (error);
2173 }
2174
2175 /* pickup the value of start_addr and osize for libraries */
2176 start_addr = addr;
2177 osize = size;
2178
2179 /*
2180 * We have our initial reservation/allocation so we need to use fixed
2181 * addresses from now on.
2182 */
2183 mflag |= MAP_FIXED;
2184
2185 mrp[0].mr_addr = addr;
2186 mrp[0].mr_msize = hdr->a_text;
2187 mrp[0].mr_fsize = hdr->a_text;
2188 mrp[0].mr_offset = 0;
2189 mrp[0].mr_prot = PROT_READ | PROT_EXEC;
2190 mrp[0].mr_flags = MR_HDR_AOUT;
2191
2192
2193 /*
2194 * Map initialized data. We are mapping over a portion of the
2195 * previous mapping which will be unmapped in VOP_MAP below.
2196 */
2197 off = P2ROUNDUP((offset_t)(hdr->a_text), PAGESIZE);
2198 msize = off;
2199 addr += off;
2200 size = hdr->a_data;
2201 error = VOP_MAP(vp, off, as, &addr, size, PROT_ALL, PROT_ALL,
2202 mflag, fcred, NULL);
2203 if (error) {
2204 (void) as_unmap(as, start_addr, osize);
2205 return (error);
2206 }
2207 msize += size;
2208 mrp[1].mr_addr = addr;
2209 mrp[1].mr_msize = size;
2210 mrp[1].mr_fsize = size;
2211 mrp[1].mr_offset = 0;
2212 mrp[1].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
2213 mrp[1].mr_flags = 0;
2214
2215 /* Need to zero out remainder of page */
2216 addr += hdr->a_data;
2217 zfoddiff = P2PHASE((size_t)addr, PAGESIZE);
2218 if (zfoddiff) {
2219 label_t ljb;
2220
2221 MOBJ_STAT_ADD(aout_zfoddiff);
2222 zfoddiff = PAGESIZE - zfoddiff;
2223 if (on_fault(&ljb)) {
2224 no_fault();
2225 MOBJ_STAT_ADD(aout_uzero_fault);
2226 (void) as_unmap(as, start_addr, osize);
2227 return (EFAULT);
2228 }
2229 uzero(addr, zfoddiff);
2230 no_fault();
2231 }
2232 msize += zfoddiff;
2233 segnum = 2;
2234
2235 /* Map bss */
2236 if (hdr->a_bss > zfoddiff) {
2237 struct segvn_crargs crargs =
2238 SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
2239 MOBJ_STAT_ADD(aout_map_bss);
2240 addr += zfoddiff;
2241 size = hdr->a_bss - zfoddiff;
2242 as_rangelock(as);
2243 (void) as_unmap(as, addr, size);
2244 error = as_map(as, addr, size, segvn_create, &crargs);
2245 as_rangeunlock(as);
2246 msize += size;
2247
2248 if (error) {
2249 MOBJ_STAT_ADD(aout_bss_fail);
2250 (void) as_unmap(as, start_addr, osize);
2251 return (error);
2252 }
2253 mrp[2].mr_addr = addr;
2254 mrp[2].mr_msize = size;
2255 mrp[2].mr_fsize = 0;
2256 mrp[2].mr_offset = 0;
2257 mrp[2].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
2258 mrp[2].mr_flags = 0;
2259
2260 addr += size;
2261 segnum = 3;
2262 }
2263
2264 /*
2265 * If we have extra bits left over, we need to include that in how
2266 * much we mapped to make sure the nlist logic is correct
2267 */
2268 msize = P2ROUNDUP(msize, PAGESIZE);
2269
2270 if (nsize && msize < nsize) {
2271 MOBJ_STAT_ADD(aout_nlist);
2272 mrp[segnum].mr_addr = addr;
2273 mrp[segnum].mr_msize = nsize - msize;
2274 mrp[segnum].mr_fsize = 0;
2275 mrp[segnum].mr_offset = 0;
2276 mrp[segnum].mr_prot = PROT_READ | PROT_EXEC;
2277 mrp[segnum].mr_flags = 0;
2278 }
2279
2280 *num_mapped = to_map;
2281 return (0);
2282 }
2283 #endif
2284
2285 /*
2286 * These are the two types of files that we can interpret and we want to read
2287 * in enough info to cover both types when looking at the initial header.
2288 */
2289 #define MAX_HEADER_SIZE (MAX(sizeof (Ehdr), sizeof (struct exec)))
2290
2291 /*
2292 * Map vp passed in in an interpreted manner. ELF and AOUT files will be
2293 * interpreted and mapped appropriately for execution.
2294 * num_mapped in - # elements in mrp
2295 * num_mapped out - # sections mapped and length of mrp array if
2296 * no errors or E2BIG returned.
2297 *
2298 * Returns 0 on success, errno value on failure.
2299 */
2300 static int
2301 mmapobj_map_interpret(vnode_t *vp, mmapobj_result_t *mrp,
2302 uint_t *num_mapped, size_t padding, cred_t *fcred)
2303 {
2304 int error = 0;
2305 vattr_t vattr;
2306 struct lib_va *lvp;
2307 caddr_t start_addr;
2308 model_t model;
2309
2310 /*
2311 * header has to be aligned to the native size of ulong_t in order
2312 * to avoid an unaligned access when dereferencing the header as
2313 * a ulong_t. Thus we allocate our array on the stack of type
2314 * ulong_t and then have header, which we dereference later as a char
2315 * array point at lheader.
2316 */
2317 ulong_t lheader[(MAX_HEADER_SIZE / (sizeof (ulong_t))) + 1];
2318 caddr_t header = (caddr_t)&lheader;
2319
2320 vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME | AT_SIZE;
2321 error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
2322 if (error) {
2323 return (error);
2324 }
2325
2326 /*
2327 * Check lib_va to see if we already have a full description
2328 * for this library. This is the fast path and only used for
2329 * ET_DYN ELF files (dynamic libraries).
2330 */
2331 if (padding == 0 && !secflag_enabled(curproc, PROC_SEC_ASLR) &&
2332 ((lvp = lib_va_find(&vattr)) != NULL)) {
2333 int num_segs;
2334
2335 model = get_udatamodel();
2336 if ((model == DATAMODEL_ILP32 &&
2337 lvp->lv_flags & LV_ELF64) ||
2338 (model == DATAMODEL_LP64 &&
2339 lvp->lv_flags & LV_ELF32)) {
2340 lib_va_release(lvp);
2341 MOBJ_STAT_ADD(fast_wrong_model);
2342 return (ENOTSUP);
2343 }
2344 num_segs = lvp->lv_num_segs;
2345 if (*num_mapped < num_segs) {
2346 *num_mapped = num_segs;
2347 lib_va_release(lvp);
2348 MOBJ_STAT_ADD(fast_e2big);
2349 return (E2BIG);
2350 }
2351
2352 /*
2353 * Check to see if we have all the mappable program headers
2354 * cached.
2355 */
2356 if (num_segs <= LIBVA_CACHED_SEGS && num_segs != 0) {
2357 MOBJ_STAT_ADD(fast);
2358 start_addr = mmapobj_lookup_start_addr(lvp);
2359 if (start_addr == NULL) {
2360 lib_va_release(lvp);
2361 return (ENOMEM);
2362 }
2363
2364 bcopy(lvp->lv_mps, mrp,
2365 num_segs * sizeof (mmapobj_result_t));
2366
2367 error = mmapobj_map_elf(vp, start_addr, mrp,
2368 num_segs, fcred, ET_DYN);
2369
2370 lib_va_release(lvp);
2371 if (error == 0) {
2372 *num_mapped = num_segs;
2373 MOBJ_STAT_ADD(fast_success);
2374 }
2375 return (error);
2376 }
2377 MOBJ_STAT_ADD(fast_not_now);
2378
2379 /* Release it for now since we'll look it up below */
2380 lib_va_release(lvp);
2381 }
2382
2383 /*
2384 * Time to see if this is a file we can interpret. If it's smaller
2385 * than this, then we can't interpret it.
2386 */
2387 if (vattr.va_size < MAX_HEADER_SIZE) {
2388 MOBJ_STAT_ADD(small_file);
2389 return (ENOTSUP);
2390 }
2391
2392 if ((error = vn_rdwr(UIO_READ, vp, header, MAX_HEADER_SIZE, 0,
2393 UIO_SYSSPACE, 0, (rlim64_t)0, fcred, NULL)) != 0) {
2394 MOBJ_STAT_ADD(read_error);
2395 return (error);
2396 }
2397
2398 /* Verify file type */
2399 if (header[EI_MAG0] == ELFMAG0 && header[EI_MAG1] == ELFMAG1 &&
2400 header[EI_MAG2] == ELFMAG2 && header[EI_MAG3] == ELFMAG3) {
2401 return (doelfwork((Ehdr *)lheader, vp, mrp, num_mapped,
2402 padding, fcred));
2403 }
2404
2405 #if defined(__sparc)
2406 /* On sparc, check for 4.X AOUT format */
2407 switch (((struct exec *)header)->a_magic) {
2408 case OMAGIC:
2409 case ZMAGIC:
2410 case NMAGIC:
2411 return (doaoutwork(vp, mrp, num_mapped,
2412 (struct exec *)lheader, fcred));
2413 }
2414 #endif
2415
2416 /* Unsupported type */
2417 MOBJ_STAT_ADD(unsupported);
2418 return (ENOTSUP);
2419 }
2420
2421 /*
2422 * Given a vnode, map it as either a flat file or interpret it and map
2423 * it according to the rules of the file type.
2424 * *num_mapped will contain the size of the mmapobj_result_t array passed in.
2425 * If padding is non-zero, the mappings will be padded by that amount
2426 * rounded up to the nearest pagesize.
2427 * If the mapping is successful, *num_mapped will contain the number of
2428 * distinct mappings created, and mrp will point to the array of
2429 * mmapobj_result_t's which describe these mappings.
2430 *
2431 * On error, -1 is returned and errno is set appropriately.
2432 * A special error case will set errno to E2BIG when there are more than
2433 * *num_mapped mappings to be created and *num_mapped will be set to the
2434 * number of mappings needed.
2435 */
2436 int
2437 mmapobj(vnode_t *vp, uint_t flags, mmapobj_result_t *mrp,
2438 uint_t *num_mapped, size_t padding, cred_t *fcred)
2439 {
2440 int to_map;
2441 int error = 0;
2442
2443 ASSERT((padding & PAGEOFFSET) == 0);
2444 ASSERT((flags & ~MMOBJ_ALL_FLAGS) == 0);
2445 ASSERT(num_mapped != NULL);
2446 ASSERT((flags & MMOBJ_PADDING) ? padding != 0 : padding == 0);
2447
2448 if ((flags & MMOBJ_INTERPRET) == 0) {
2449 to_map = padding ? 3 : 1;
2450 if (*num_mapped < to_map) {
2451 *num_mapped = to_map;
2452 MOBJ_STAT_ADD(flat_e2big);
2453 return (E2BIG);
2454 }
2455 error = mmapobj_map_flat(vp, mrp, padding, fcred);
2456
2457 if (error) {
2458 return (error);
2459 }
2460 *num_mapped = to_map;
2461 return (0);
2462 }
2463
2464 error = mmapobj_map_interpret(vp, mrp, num_mapped, padding, fcred);
2465 return (error);
2466 }