1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  * Copyright 2014 Joyent, Inc.  All rights reserved.
  25  */
  26 
  27 #include <sys/types.h>
  28 #include <sys/sysmacros.h>
  29 #include <sys/kmem.h>
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/errno.h>
  33 #include <sys/mman.h>
  34 #include <sys/cmn_err.h>
  35 #include <sys/cred.h>
  36 #include <sys/vmsystm.h>
  37 #include <sys/machsystm.h>
  38 #include <sys/debug.h>
  39 #include <vm/as.h>
  40 #include <vm/seg.h>
  41 #include <sys/vmparam.h>
  42 #include <sys/vfs.h>
  43 #include <sys/elf.h>
  44 #include <sys/machelf.h>
  45 #include <sys/corectl.h>
  46 #include <sys/exec.h>
  47 #include <sys/exechdr.h>
  48 #include <sys/autoconf.h>
  49 #include <sys/mem.h>
  50 #include <vm/seg_dev.h>
  51 #include <sys/vmparam.h>
  52 #include <sys/mmapobj.h>
  53 #include <sys/atomic.h>
  54 
  55 /*
  56  * Theory statement:
  57  *
  58  * The main driving force behind mmapobj is to interpret and map ELF files
  59  * inside of the kernel instead of having the linker be responsible for this.
  60  *
  61  * mmapobj also supports the AOUT 4.x binary format as well as flat files in
  62  * a read only manner.
  63  *
  64  * When interpreting and mapping an ELF file, mmapobj will map each PT_LOAD
  65  * or PT_SUNWBSS segment according to the ELF standard.  Refer to the "Linker
  66  * and Libraries Guide" for more information about the standard and mapping
  67  * rules.
  68  *
  69  * Having mmapobj interpret and map objects will allow the kernel to make the
  70  * best decision for where to place the mappings for said objects.  Thus, we
  71  * can make optimizations inside of the kernel for specific platforms or cache
  72  * mapping information to make mapping objects faster.  The cache is ignored
  73  * if ASLR is enabled.
  74  *
  75  * The lib_va_hash will be one such optimization.  For each ELF object that
  76  * mmapobj is asked to interpret, we will attempt to cache the information
  77  * about the PT_LOAD and PT_SUNWBSS sections to speed up future mappings of
  78  * the same objects.  We will cache up to LIBVA_CACHED_SEGS (see below) program
  79  * headers which should cover a majority of the libraries out there without
  80  * wasting space.  In order to make sure that the cached information is valid,
  81  * we check the passed in vnode's mtime and ctime to make sure the vnode
  82  * has not been modified since the last time we used it.
  83  *
  84  * In addition, the lib_va_hash may contain a preferred starting VA for the
  85  * object which can be useful for platforms which support a shared context.
  86  * This will increase the likelyhood that library text can be shared among
  87  * many different processes.  We limit the reserved VA space for 32 bit objects
  88  * in order to minimize fragmenting the processes address space.
  89  *
  90  * In addition to the above, the mmapobj interface allows for padding to be
  91  * requested before the first mapping and after the last mapping created.
  92  * When padding is requested, no additional optimizations will be made for
  93  * that request.
  94  */
  95 
  96 /*
  97  * Threshold to prevent allocating too much kernel memory to read in the
  98  * program headers for an object.  If it requires more than below,
  99  * we will use a KM_NOSLEEP allocation to allocate memory to hold all of the
 100  * program headers which could possibly fail.  If less memory than below is
 101  * needed, then we use a KM_SLEEP allocation and are willing to wait for the
 102  * memory if we need to.
 103  */
 104 size_t mmapobj_alloc_threshold = 65536;
 105 
 106 /* Debug stats for test coverage */
 107 #ifdef DEBUG
 108 struct mobj_stats {
 109         uint_t  mobjs_unmap_called;
 110         uint_t  mobjs_remap_devnull;
 111         uint_t  mobjs_lookup_start;
 112         uint_t  mobjs_alloc_start;
 113         uint_t  mobjs_alloc_vmem;
 114         uint_t  mobjs_add_collision;
 115         uint_t  mobjs_get_addr;
 116         uint_t  mobjs_map_flat_no_padding;
 117         uint_t  mobjs_map_flat_padding;
 118         uint_t  mobjs_map_ptload_text;
 119         uint_t  mobjs_map_ptload_initdata;
 120         uint_t  mobjs_map_ptload_preread;
 121         uint_t  mobjs_map_ptload_unaligned_text;
 122         uint_t  mobjs_map_ptload_unaligned_map_fail;
 123         uint_t  mobjs_map_ptload_unaligned_read_fail;
 124         uint_t  mobjs_zfoddiff;
 125         uint_t  mobjs_zfoddiff_nowrite;
 126         uint_t  mobjs_zfodextra;
 127         uint_t  mobjs_ptload_failed;
 128         uint_t  mobjs_map_elf_no_holes;
 129         uint_t  mobjs_unmap_hole;
 130         uint_t  mobjs_nomem_header;
 131         uint_t  mobjs_inval_header;
 132         uint_t  mobjs_overlap_header;
 133         uint_t  mobjs_np2_align;
 134         uint_t  mobjs_np2_align_overflow;
 135         uint_t  mobjs_exec_padding;
 136         uint_t  mobjs_exec_addr_mapped;
 137         uint_t  mobjs_exec_addr_devnull;
 138         uint_t  mobjs_exec_addr_in_use;
 139         uint_t  mobjs_lvp_found;
 140         uint_t  mobjs_no_loadable_yet;
 141         uint_t  mobjs_nothing_to_map;
 142         uint_t  mobjs_e2big;
 143         uint_t  mobjs_dyn_pad_align;
 144         uint_t  mobjs_dyn_pad_noalign;
 145         uint_t  mobjs_alloc_start_fail;
 146         uint_t  mobjs_lvp_nocache;
 147         uint_t  mobjs_extra_padding;
 148         uint_t  mobjs_lvp_not_needed;
 149         uint_t  mobjs_no_mem_map_sz;
 150         uint_t  mobjs_check_exec_failed;
 151         uint_t  mobjs_lvp_used;
 152         uint_t  mobjs_wrong_model;
 153         uint_t  mobjs_noexec_fs;
 154         uint_t  mobjs_e2big_et_rel;
 155         uint_t  mobjs_et_rel_mapped;
 156         uint_t  mobjs_unknown_elf_type;
 157         uint_t  mobjs_phent32_too_small;
 158         uint_t  mobjs_phent64_too_small;
 159         uint_t  mobjs_inval_elf_class;
 160         uint_t  mobjs_too_many_phdrs;
 161         uint_t  mobjs_no_phsize;
 162         uint_t  mobjs_phsize_large;
 163         uint_t  mobjs_phsize_xtralarge;
 164         uint_t  mobjs_fast_wrong_model;
 165         uint_t  mobjs_fast_e2big;
 166         uint_t  mobjs_fast;
 167         uint_t  mobjs_fast_success;
 168         uint_t  mobjs_fast_not_now;
 169         uint_t  mobjs_small_file;
 170         uint_t  mobjs_read_error;
 171         uint_t  mobjs_unsupported;
 172         uint_t  mobjs_flat_e2big;
 173         uint_t  mobjs_phent_align32;
 174         uint_t  mobjs_phent_align64;
 175         uint_t  mobjs_lib_va_find_hit;
 176         uint_t  mobjs_lib_va_find_delay_delete;
 177         uint_t  mobjs_lib_va_find_delete;
 178         uint_t  mobjs_lib_va_add_delay_delete;
 179         uint_t  mobjs_lib_va_add_delete;
 180         uint_t  mobjs_lib_va_create_failure;
 181         uint_t  mobjs_min_align;
 182 #if defined(__sparc)
 183         uint_t  mobjs_aout_uzero_fault;
 184         uint_t  mobjs_aout_64bit_try;
 185         uint_t  mobjs_aout_noexec;
 186         uint_t  mobjs_aout_e2big;
 187         uint_t  mobjs_aout_lib;
 188         uint_t  mobjs_aout_fixed;
 189         uint_t  mobjs_aout_zfoddiff;
 190         uint_t  mobjs_aout_map_bss;
 191         uint_t  mobjs_aout_bss_fail;
 192         uint_t  mobjs_aout_nlist;
 193         uint_t  mobjs_aout_addr_in_use;
 194 #endif
 195 } mobj_stats;
 196 
 197 #define MOBJ_STAT_ADD(stat)             ((mobj_stats.mobjs_##stat)++)
 198 #else
 199 #define MOBJ_STAT_ADD(stat)
 200 #endif
 201 
 202 /*
 203  * Check if addr is at or above the address space reserved for the stack.
 204  * The stack is at the top of the address space for all sparc processes
 205  * and 64 bit x86 processes.  For 32 bit x86, the stack is not at the top
 206  * of the address space and thus this check wil always return false for
 207  * 32 bit x86 processes.
 208  */
 209 #if defined(__sparc)
 210 #define OVERLAPS_STACK(addr, p)                                         \
 211         (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK)))
 212 #elif defined(__amd64)
 213 #define OVERLAPS_STACK(addr, p)                                         \
 214         ((p->p_model == DATAMODEL_LP64) &&                           \
 215         (addr >= (p->p_usrstack - ((p->p_stk_ctl + PAGEOFFSET) & PAGEMASK))))
 216 #elif defined(__i386)
 217 #define OVERLAPS_STACK(addr, p) 0
 218 #endif
 219 
 220 /* lv_flags values - bitmap */
 221 #define LV_ELF32        0x1             /* 32 bit ELF file */
 222 #define LV_ELF64        0x2             /* 64 bit ELF file */
 223 #define LV_DEL          0x4             /* delete when lv_refcnt hits zero */
 224 
 225 /*
 226  * Note: lv_num_segs will denote how many segments this file has and will
 227  * only be set after the lv_mps array has been filled out.
 228  * lv_mps can only be valid if lv_num_segs is non-zero.
 229  */
 230 struct lib_va {
 231         struct lib_va           *lv_next;
 232         caddr_t                 lv_base_va;     /* start va for library */
 233         ssize_t                 lv_len;         /* total va span of library */
 234         size_t                  lv_align;       /* minimum alignment */
 235         uint64_t                lv_nodeid;      /* filesystem node id */
 236         uint64_t                lv_fsid;        /* filesystem id */
 237         timestruc_t             lv_ctime;       /* last time file was changed */
 238         timestruc_t             lv_mtime;       /* or modified */
 239         mmapobj_result_t        lv_mps[LIBVA_CACHED_SEGS]; /* cached pheaders */
 240         int                     lv_num_segs;    /* # segs for this file */
 241         int                     lv_flags;
 242         uint_t                  lv_refcnt;      /* number of holds on struct */
 243 };
 244 
 245 #define LIB_VA_SIZE     1024
 246 #define LIB_VA_MASK     (LIB_VA_SIZE - 1)
 247 #define LIB_VA_MUTEX_SHIFT      3
 248 
 249 #if (LIB_VA_SIZE & (LIB_VA_SIZE - 1))
 250 #error  "LIB_VA_SIZE is not a power of 2"
 251 #endif
 252 
 253 static struct lib_va *lib_va_hash[LIB_VA_SIZE];
 254 static kmutex_t lib_va_hash_mutex[LIB_VA_SIZE >> LIB_VA_MUTEX_SHIFT];
 255 
 256 #define LIB_VA_HASH_MUTEX(index)                                        \
 257         (&lib_va_hash_mutex[index >> LIB_VA_MUTEX_SHIFT])
 258 
 259 #define LIB_VA_HASH(nodeid)                                             \
 260         (((nodeid) ^ ((nodeid) << 7) ^ ((nodeid) << 13)) & LIB_VA_MASK)
 261 
 262 #define LIB_VA_MATCH_ID(arg1, arg2)                                     \
 263         ((arg1)->lv_nodeid == (arg2)->va_nodeid &&                        \
 264         (arg1)->lv_fsid == (arg2)->va_fsid)
 265 
 266 #define LIB_VA_MATCH_TIME(arg1, arg2)                                   \
 267         ((arg1)->lv_ctime.tv_sec == (arg2)->va_ctime.tv_sec &&            \
 268         (arg1)->lv_mtime.tv_sec == (arg2)->va_mtime.tv_sec &&             \
 269         (arg1)->lv_ctime.tv_nsec == (arg2)->va_ctime.tv_nsec &&           \
 270         (arg1)->lv_mtime.tv_nsec == (arg2)->va_mtime.tv_nsec)
 271 
 272 #define LIB_VA_MATCH(arg1, arg2)                                        \
 273         (LIB_VA_MATCH_ID(arg1, arg2) && LIB_VA_MATCH_TIME(arg1, arg2))
 274 
 275 /*
 276  * lib_va will be used for optimized allocation of address ranges for
 277  * libraries, such that subsequent mappings of the same library will attempt
 278  * to use the same VA as previous mappings of that library.
 279  * In order to map libraries at the same VA in many processes, we need to carve
 280  * out our own address space for them which is unique across many processes.
 281  * We use different arenas for 32 bit and 64 bit libraries.
 282  *
 283  * Since the 32 bit address space is relatively small, we limit the number of
 284  * libraries which try to use consistent virtual addresses to lib_threshold.
 285  * For 64 bit libraries there is no such limit since the address space is large.
 286  */
 287 static vmem_t *lib_va_32_arena;
 288 static vmem_t *lib_va_64_arena;
 289 uint_t lib_threshold = 20;      /* modifiable via /etc/system */
 290 
 291 static kmutex_t lib_va_init_mutex;      /* no need to initialize */
 292 
 293 /*
 294  * Number of 32 bit and 64 bit libraries in lib_va hash.
 295  */
 296 static uint_t libs_mapped_32 = 0;
 297 static uint_t libs_mapped_64 = 0;
 298 
 299 /*
 300  * Free up the resources associated with lvp as well as lvp itself.
 301  * We also decrement the number of libraries mapped via a lib_va
 302  * cached virtual address.
 303  */
 304 void
 305 lib_va_free(struct lib_va *lvp)
 306 {
 307         int is_64bit = lvp->lv_flags & LV_ELF64;
 308         ASSERT(lvp->lv_refcnt == 0);
 309 
 310         if (lvp->lv_base_va != NULL) {
 311                 vmem_xfree(is_64bit ? lib_va_64_arena : lib_va_32_arena,
 312                     lvp->lv_base_va, lvp->lv_len);
 313                 if (is_64bit) {
 314                         atomic_dec_32(&libs_mapped_64);
 315                 } else {
 316                         atomic_dec_32(&libs_mapped_32);
 317                 }
 318         }
 319         kmem_free(lvp, sizeof (struct lib_va));
 320 }
 321 
 322 /*
 323  * See if the file associated with the vap passed in is in the lib_va hash.
 324  * If it is and the file has not been modified since last use, then
 325  * return a pointer to that data.  Otherwise, return NULL if the file has
 326  * changed or the file was not found in the hash.
 327  */
 328 static struct lib_va *
 329 lib_va_find(vattr_t *vap)
 330 {
 331         struct lib_va *lvp;
 332         struct lib_va *del = NULL;
 333         struct lib_va **tmp;
 334         uint_t index;
 335         index = LIB_VA_HASH(vap->va_nodeid);
 336 
 337         mutex_enter(LIB_VA_HASH_MUTEX(index));
 338         tmp = &lib_va_hash[index];
 339         while (*tmp != NULL) {
 340                 lvp = *tmp;
 341                 if (LIB_VA_MATCH_ID(lvp, vap)) {
 342                         if (LIB_VA_MATCH_TIME(lvp, vap)) {
 343                                 ASSERT((lvp->lv_flags & LV_DEL) == 0);
 344                                 lvp->lv_refcnt++;
 345                                 MOBJ_STAT_ADD(lib_va_find_hit);
 346                         } else {
 347                                 /*
 348                                  * file was updated since last use.
 349                                  * need to remove it from list.
 350                                  */
 351                                 del = lvp;
 352                                 *tmp = del->lv_next;
 353                                 del->lv_next = NULL;
 354                                 /*
 355                                  * If we can't delete it now, mark it for later
 356                                  */
 357                                 if (del->lv_refcnt) {
 358                                         MOBJ_STAT_ADD(lib_va_find_delay_delete);
 359                                         del->lv_flags |= LV_DEL;
 360                                         del = NULL;
 361                                 }
 362                                 lvp = NULL;
 363                         }
 364                         mutex_exit(LIB_VA_HASH_MUTEX(index));
 365                         if (del) {
 366                                 ASSERT(del->lv_refcnt == 0);
 367                                 MOBJ_STAT_ADD(lib_va_find_delete);
 368                                 lib_va_free(del);
 369                         }
 370                         return (lvp);
 371                 }
 372                 tmp = &lvp->lv_next;
 373         }
 374         mutex_exit(LIB_VA_HASH_MUTEX(index));
 375         return (NULL);
 376 }
 377 
 378 /*
 379  * Add a new entry to the lib_va hash.
 380  * Search the hash while holding the appropriate mutex to make sure that the
 381  * data is not already in the cache.  If we find data that is in the cache
 382  * already and has not been modified since last use, we return NULL.  If it
 383  * has been modified since last use, we will remove that entry from
 384  * the hash and it will be deleted once it's reference count reaches zero.
 385  * If there is no current entry in the hash we will add the new entry and
 386  * return it to the caller who is responsible for calling lib_va_release to
 387  * drop their reference count on it.
 388  *
 389  * lv_num_segs will be set to zero since the caller needs to add that
 390  * information to the data structure.
 391  */
 392 static struct lib_va *
 393 lib_va_add_hash(caddr_t base_va, ssize_t len, size_t align, vattr_t *vap)
 394 {
 395         struct lib_va *lvp;
 396         uint_t index;
 397         model_t model;
 398         struct lib_va **tmp;
 399         struct lib_va *del = NULL;
 400 
 401         model = get_udatamodel();
 402         index = LIB_VA_HASH(vap->va_nodeid);
 403 
 404         lvp = kmem_alloc(sizeof (struct lib_va), KM_SLEEP);
 405 
 406         mutex_enter(LIB_VA_HASH_MUTEX(index));
 407 
 408         /*
 409          * Make sure not adding same data a second time.
 410          * The hash chains should be relatively short and adding
 411          * is a relatively rare event, so it's worth the check.
 412          */
 413         tmp = &lib_va_hash[index];
 414         while (*tmp != NULL) {
 415                 if (LIB_VA_MATCH_ID(*tmp, vap)) {
 416                         if (LIB_VA_MATCH_TIME(*tmp, vap)) {
 417                                 mutex_exit(LIB_VA_HASH_MUTEX(index));
 418                                 kmem_free(lvp, sizeof (struct lib_va));
 419                                 return (NULL);
 420                         }
 421 
 422                         /*
 423                          * We have the same nodeid and fsid but the file has
 424                          * been modified since we last saw it.
 425                          * Need to remove the old node and add this new
 426                          * one.
 427                          * Could probably use a callback mechanism to make
 428                          * this cleaner.
 429                          */
 430                         ASSERT(del == NULL);
 431                         del = *tmp;
 432                         *tmp = del->lv_next;
 433                         del->lv_next = NULL;
 434 
 435                         /*
 436                          * Check to see if we can free it.  If lv_refcnt
 437                          * is greater than zero, than some other thread
 438                          * has a reference to the one we want to delete
 439                          * and we can not delete it.  All of this is done
 440                          * under the lib_va_hash_mutex lock so it is atomic.
 441                          */
 442                         if (del->lv_refcnt) {
 443                                 MOBJ_STAT_ADD(lib_va_add_delay_delete);
 444                                 del->lv_flags |= LV_DEL;
 445                                 del = NULL;
 446                         }
 447                         /* tmp is already advanced */
 448                         continue;
 449                 }
 450                 tmp = &((*tmp)->lv_next);
 451         }
 452 
 453         lvp->lv_base_va = base_va;
 454         lvp->lv_len = len;
 455         lvp->lv_align = align;
 456         lvp->lv_nodeid = vap->va_nodeid;
 457         lvp->lv_fsid = vap->va_fsid;
 458         lvp->lv_ctime.tv_sec = vap->va_ctime.tv_sec;
 459         lvp->lv_ctime.tv_nsec = vap->va_ctime.tv_nsec;
 460         lvp->lv_mtime.tv_sec = vap->va_mtime.tv_sec;
 461         lvp->lv_mtime.tv_nsec = vap->va_mtime.tv_nsec;
 462         lvp->lv_next = NULL;
 463         lvp->lv_refcnt = 1;
 464 
 465         /* Caller responsible for filling this and lv_mps out */
 466         lvp->lv_num_segs = 0;
 467 
 468         if (model == DATAMODEL_LP64) {
 469                 lvp->lv_flags = LV_ELF64;
 470         } else {
 471                 ASSERT(model == DATAMODEL_ILP32);
 472                 lvp->lv_flags = LV_ELF32;
 473         }
 474 
 475         if (base_va != NULL) {
 476                 if (model == DATAMODEL_LP64) {
 477                         atomic_inc_32(&libs_mapped_64);
 478                 } else {
 479                         ASSERT(model == DATAMODEL_ILP32);
 480                         atomic_inc_32(&libs_mapped_32);
 481                 }
 482         }
 483         ASSERT(*tmp == NULL);
 484         *tmp = lvp;
 485         mutex_exit(LIB_VA_HASH_MUTEX(index));
 486         if (del) {
 487                 ASSERT(del->lv_refcnt == 0);
 488                 MOBJ_STAT_ADD(lib_va_add_delete);
 489                 lib_va_free(del);
 490         }
 491         return (lvp);
 492 }
 493 
 494 /*
 495  * Release the hold on lvp which was acquired by lib_va_find or lib_va_add_hash.
 496  * In addition, if this is the last hold and lvp is marked for deletion,
 497  * free up it's reserved address space and free the structure.
 498  */
 499 static void
 500 lib_va_release(struct lib_va *lvp)
 501 {
 502         uint_t index;
 503         int to_del = 0;
 504 
 505         ASSERT(lvp->lv_refcnt > 0);
 506 
 507         index = LIB_VA_HASH(lvp->lv_nodeid);
 508         mutex_enter(LIB_VA_HASH_MUTEX(index));
 509         if (--lvp->lv_refcnt == 0 && (lvp->lv_flags & LV_DEL)) {
 510                 to_del = 1;
 511         }
 512         mutex_exit(LIB_VA_HASH_MUTEX(index));
 513         if (to_del) {
 514                 ASSERT(lvp->lv_next == 0);
 515                 lib_va_free(lvp);
 516         }
 517 }
 518 
 519 /*
 520  * Dummy function for mapping through /dev/null
 521  * Normally I would have used mmmmap in common/io/mem.c
 522  * but that is a static function, and for /dev/null, it
 523  * just returns -1.
 524  */
 525 /* ARGSUSED */
 526 static int
 527 mmapobj_dummy(dev_t dev, off_t off, int prot)
 528 {
 529         return (-1);
 530 }
 531 
 532 /*
 533  * Called when an error occurred which requires mmapobj to return failure.
 534  * All mapped objects will be unmapped and /dev/null mappings will be
 535  * reclaimed if necessary.
 536  * num_mapped is the number of elements of mrp which have been mapped, and
 537  * num_segs is the total number of elements in mrp.
 538  * For e_type ET_EXEC, we need to unmap all of the elements in mrp since
 539  * we had already made reservations for them.
 540  * If num_mapped equals num_segs, then we know that we had fully mapped
 541  * the file and only need to clean up the segments described.
 542  * If they are not equal, then for ET_DYN we will unmap the range from the
 543  * end of the last mapped segment to the end of the last segment in mrp
 544  * since we would have made a reservation for that memory earlier.
 545  * If e_type is passed in as zero, num_mapped must equal num_segs.
 546  */
 547 void
 548 mmapobj_unmap(mmapobj_result_t *mrp, int num_mapped, int num_segs,
 549     ushort_t e_type)
 550 {
 551         int i;
 552         struct as *as = curproc->p_as;
 553         caddr_t addr;
 554         size_t size;
 555 
 556         if (e_type == ET_EXEC) {
 557                 num_mapped = num_segs;
 558         }
 559 #ifdef DEBUG
 560         if (e_type == 0) {
 561                 ASSERT(num_mapped == num_segs);
 562         }
 563 #endif
 564 
 565         MOBJ_STAT_ADD(unmap_called);
 566         for (i = 0; i < num_mapped; i++) {
 567 
 568                 /*
 569                  * If we are going to have to create a mapping we need to
 570                  * make sure that no one else will use the address we
 571                  * need to remap between the time it is unmapped and
 572                  * mapped below.
 573                  */
 574                 if (mrp[i].mr_flags & MR_RESV) {
 575                         as_rangelock(as);
 576                 }
 577                 /* Always need to unmap what we mapped */
 578                 (void) as_unmap(as, mrp[i].mr_addr, mrp[i].mr_msize);
 579 
 580                 /* Need to reclaim /dev/null reservation from earlier */
 581                 if (mrp[i].mr_flags & MR_RESV) {
 582                         struct segdev_crargs dev_a;
 583 
 584                         ASSERT(e_type != ET_DYN);
 585                         /*
 586                          * Use seg_dev segment driver for /dev/null mapping.
 587                          */
 588                         dev_a.mapfunc = mmapobj_dummy;
 589                         dev_a.dev = makedevice(mm_major, M_NULL);
 590                         dev_a.offset = 0;
 591                         dev_a.type = 0;         /* neither PRIVATE nor SHARED */
 592                         dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE;
 593                         dev_a.hat_attr = 0;
 594                         dev_a.hat_flags = 0;
 595 
 596                         (void) as_map(as, mrp[i].mr_addr, mrp[i].mr_msize,
 597                             segdev_create, &dev_a);
 598                         MOBJ_STAT_ADD(remap_devnull);
 599                         as_rangeunlock(as);
 600                 }
 601         }
 602 
 603         if (num_mapped != num_segs) {
 604                 ASSERT(e_type == ET_DYN);
 605                 /* Need to unmap any reservation made after last mapped seg */
 606                 if (num_mapped == 0) {
 607                         addr = mrp[0].mr_addr;
 608                 } else {
 609                         addr = mrp[num_mapped - 1].mr_addr +
 610                             mrp[num_mapped - 1].mr_msize;
 611                 }
 612                 size = (size_t)mrp[num_segs - 1].mr_addr +
 613                     mrp[num_segs - 1].mr_msize - (size_t)addr;
 614                 (void) as_unmap(as, addr, size);
 615 
 616                 /*
 617                  * Now we need to unmap the holes between mapped segs.
 618                  * Note that we have not mapped all of the segments and thus
 619                  * the holes between segments would not have been unmapped
 620                  * yet.  If num_mapped == num_segs, then all of the holes
 621                  * between segments would have already been unmapped.
 622                  */
 623 
 624                 for (i = 1; i < num_mapped; i++) {
 625                         addr = mrp[i - 1].mr_addr + mrp[i - 1].mr_msize;
 626                         size = mrp[i].mr_addr - addr;
 627                         (void) as_unmap(as, addr, size);
 628                 }
 629         }
 630 }
 631 
 632 /*
 633  * We need to add the start address into mrp so that the unmap function
 634  * has absolute addresses to use.
 635  */
 636 static void
 637 mmapobj_unmap_exec(mmapobj_result_t *mrp, int num_mapped, caddr_t start_addr)
 638 {
 639         int i;
 640 
 641         for (i = 0; i < num_mapped; i++) {
 642                 mrp[i].mr_addr += (size_t)start_addr;
 643         }
 644         mmapobj_unmap(mrp, num_mapped, num_mapped, ET_EXEC);
 645 }
 646 
 647 static caddr_t
 648 mmapobj_lookup_start_addr(struct lib_va *lvp)
 649 {
 650         proc_t *p = curproc;
 651         struct as *as = p->p_as;
 652         struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
 653         int error;
 654         uint_t ma_flags = _MAP_LOW32;
 655         caddr_t base = NULL;
 656         size_t len;
 657         size_t align;
 658 
 659         ASSERT(lvp != NULL);
 660         MOBJ_STAT_ADD(lookup_start);
 661 
 662         as_rangelock(as);
 663 
 664         base = lvp->lv_base_va;
 665         len = lvp->lv_len;
 666 
 667         /*
 668          * If we don't have an expected base address, or the one that we want
 669          * to use is not available or acceptable, go get an acceptable
 670          * address range.
 671          */
 672         if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
 673             valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
 674             RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
 675                 if (lvp->lv_flags & LV_ELF64) {
 676                         ma_flags = 0;
 677                 }
 678 
 679                 align = lvp->lv_align;
 680                 if (align > 1) {
 681                         ma_flags |= MAP_ALIGN;
 682                 }
 683 
 684                 base = (caddr_t)align;
 685                 map_addr(&base, len, 0, 1, ma_flags);
 686         }
 687 
 688         /*
 689          * Need to reserve the address space we're going to use.
 690          * Don't reserve swap space since we'll be mapping over this.
 691          */
 692         if (base != NULL) {
 693                 crargs.flags |= MAP_NORESERVE;
 694                 error = as_map(as, base, len, segvn_create, &crargs);
 695                 if (error) {
 696                         base = NULL;
 697                 }
 698         }
 699 
 700         as_rangeunlock(as);
 701         return (base);
 702 }
 703 
 704 /*
 705  * Get the starting address for a given file to be mapped and return it
 706  * to the caller.  If we're using lib_va and we need to allocate an address,
 707  * we will attempt to allocate it from the global reserved pool such that the
 708  * same address can be used in the future for this file.  If we can't use the
 709  * reserved address then we just get one that will fit in our address space.
 710  *
 711  * Returns the starting virtual address for the range to be mapped or NULL
 712  * if an error is encountered. If we successfully insert the requested info
 713  * into the lib_va hash, then *lvpp will be set to point to this lib_va
 714  * structure.  The structure will have a hold on it and thus lib_va_release
 715  * needs to be called on it by the caller.  This function will not fill out
 716  * lv_mps or lv_num_segs since it does not have enough information to do so.
 717  * The caller is responsible for doing this making sure that any modifications
 718  * to lv_mps are visible before setting lv_num_segs.
 719  */
 720 static caddr_t
 721 mmapobj_alloc_start_addr(struct lib_va **lvpp, size_t len, int use_lib_va,
 722     int randomize, size_t align, vattr_t *vap)
 723 {
 724         proc_t *p = curproc;
 725         struct as *as = p->p_as;
 726         struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
 727         int error;
 728         model_t model;
 729         uint_t ma_flags = _MAP_LOW32;
 730         caddr_t base = NULL;
 731         vmem_t *model_vmem;
 732         size_t lib_va_start;
 733         size_t lib_va_end;
 734         size_t lib_va_len;
 735 
 736         ASSERT(lvpp != NULL);
 737         ASSERT((randomize & use_lib_va) != 1);
 738 
 739         MOBJ_STAT_ADD(alloc_start);
 740         model = get_udatamodel();
 741 
 742         if (model == DATAMODEL_LP64) {
 743                 ma_flags = 0;
 744                 model_vmem = lib_va_64_arena;
 745         } else {
 746                 ASSERT(model == DATAMODEL_ILP32);
 747                 model_vmem = lib_va_32_arena;
 748         }
 749 
 750         if (align > 1) {
 751                 ma_flags |= MAP_ALIGN;
 752         }
 753 
 754         if (randomize != 0)
 755                 ma_flags |= _MAP_RANDOMIZE;
 756 
 757         if (use_lib_va) {
 758                 /*
 759                  * The first time through, we need to setup the lib_va arenas.
 760                  * We call map_addr to find a suitable range of memory to map
 761                  * the given library, and we will set the highest address
 762                  * in our vmem arena to the end of this adddress range.
 763                  * We allow up to half of the address space to be used
 764                  * for lib_va addresses but we do not prevent any allocations
 765                  * in this range from other allocation paths.
 766                  */
 767                 if (lib_va_64_arena == NULL && model == DATAMODEL_LP64) {
 768                         mutex_enter(&lib_va_init_mutex);
 769                         if (lib_va_64_arena == NULL) {
 770                                 base = (caddr_t)align;
 771                                 as_rangelock(as);
 772                                 map_addr(&base, len, 0, 1, ma_flags);
 773                                 as_rangeunlock(as);
 774                                 if (base == NULL) {
 775                                         mutex_exit(&lib_va_init_mutex);
 776                                         MOBJ_STAT_ADD(lib_va_create_failure);
 777                                         goto nolibva;
 778                                 }
 779                                 lib_va_end = (size_t)base + len;
 780                                 lib_va_len = lib_va_end >> 1;
 781                                 lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
 782                                 lib_va_start = lib_va_end - lib_va_len;
 783 
 784                                 /*
 785                                  * Need to make sure we avoid the address hole.
 786                                  * We know lib_va_end is valid but we need to
 787                                  * make sure lib_va_start is as well.
 788                                  */
 789                                 if ((lib_va_end > (size_t)hole_end) &&
 790                                     (lib_va_start < (size_t)hole_end)) {
 791                                         lib_va_start = P2ROUNDUP(
 792                                             (size_t)hole_end, PAGESIZE);
 793                                         lib_va_len = lib_va_end - lib_va_start;
 794                                 }
 795                                 lib_va_64_arena = vmem_create("lib_va_64",
 796                                     (void *)lib_va_start, lib_va_len, PAGESIZE,
 797                                     NULL, NULL, NULL, 0,
 798                                     VM_NOSLEEP | VMC_IDENTIFIER);
 799                                 if (lib_va_64_arena == NULL) {
 800                                         mutex_exit(&lib_va_init_mutex);
 801                                         goto nolibva;
 802                                 }
 803                         }
 804                         model_vmem = lib_va_64_arena;
 805                         mutex_exit(&lib_va_init_mutex);
 806                 } else if (lib_va_32_arena == NULL &&
 807                     model == DATAMODEL_ILP32) {
 808                         mutex_enter(&lib_va_init_mutex);
 809                         if (lib_va_32_arena == NULL) {
 810                                 base = (caddr_t)align;
 811                                 as_rangelock(as);
 812                                 map_addr(&base, len, 0, 1, ma_flags);
 813                                 as_rangeunlock(as);
 814                                 if (base == NULL) {
 815                                         mutex_exit(&lib_va_init_mutex);
 816                                         MOBJ_STAT_ADD(lib_va_create_failure);
 817                                         goto nolibva;
 818                                 }
 819                                 lib_va_end = (size_t)base + len;
 820                                 lib_va_len = lib_va_end >> 1;
 821                                 lib_va_len = P2ROUNDUP(lib_va_len, PAGESIZE);
 822                                 lib_va_start = lib_va_end - lib_va_len;
 823                                 lib_va_32_arena = vmem_create("lib_va_32",
 824                                     (void *)lib_va_start, lib_va_len, PAGESIZE,
 825                                     NULL, NULL, NULL, 0,
 826                                     VM_NOSLEEP | VMC_IDENTIFIER);
 827                                 if (lib_va_32_arena == NULL) {
 828                                         mutex_exit(&lib_va_init_mutex);
 829                                         goto nolibva;
 830                                 }
 831                         }
 832                         model_vmem = lib_va_32_arena;
 833                         mutex_exit(&lib_va_init_mutex);
 834                 }
 835 
 836                 if (model == DATAMODEL_LP64 || libs_mapped_32 < lib_threshold) {
 837                         base = vmem_xalloc(model_vmem, len, align, 0, 0, NULL,
 838                             NULL, VM_NOSLEEP | VM_ENDALLOC);
 839                         MOBJ_STAT_ADD(alloc_vmem);
 840                 }
 841 
 842                 /*
 843                  * Even if the address fails to fit in our address space,
 844                  * or we can't use a reserved address,
 845                  * we should still save it off in lib_va_hash.
 846                  */
 847                 *lvpp = lib_va_add_hash(base, len, align, vap);
 848 
 849                 /*
 850                  * Check for collision on insertion and free up our VA space.
 851                  * This is expected to be rare, so we'll just reset base to
 852                  * NULL instead of looking it up in the lib_va hash.
 853                  */
 854                 if (*lvpp == NULL) {
 855                         if (base != NULL) {
 856                                 vmem_xfree(model_vmem, base, len);
 857                                 base = NULL;
 858                                 MOBJ_STAT_ADD(add_collision);
 859                         }
 860                 }
 861         }
 862 
 863 nolibva:
 864         as_rangelock(as);
 865 
 866         /*
 867          * If we don't have an expected base address, or the one that we want
 868          * to use is not available or acceptable, go get an acceptable
 869          * address range.
 870          *
 871          * If ASLR is enabled, we should never have used the cache, and should
 872          * also start our real work here, in the consequent of the next
 873          * condition.
 874          */
 875         if (randomize != 0)
 876                 ASSERT(base == NULL);
 877 
 878         if (base == NULL || as_gap(as, len, &base, &len, 0, NULL) ||
 879             valid_usr_range(base, len, PROT_ALL, as, as->a_userlimit) !=
 880             RANGE_OKAY || OVERLAPS_STACK(base + len, p)) {
 881                 MOBJ_STAT_ADD(get_addr);
 882                 base = (caddr_t)align;
 883                 map_addr(&base, len, 0, 1, ma_flags);
 884         }
 885 
 886         /*
 887          * Need to reserve the address space we're going to use.
 888          * Don't reserve swap space since we'll be mapping over this.
 889          */
 890         if (base != NULL) {
 891                 /* Don't reserve swap space since we'll be mapping over this */
 892                 crargs.flags |= MAP_NORESERVE;
 893                 error = as_map(as, base, len, segvn_create, &crargs);
 894                 if (error) {
 895                         base = NULL;
 896                 }
 897         }
 898 
 899         as_rangeunlock(as);
 900         return (base);
 901 }
 902 
 903 /*
 904  * Map the file associated with vp into the address space as a single
 905  * read only private mapping.
 906  * Returns 0 for success, and non-zero for failure to map the file.
 907  */
 908 static int
 909 mmapobj_map_flat(vnode_t *vp, mmapobj_result_t *mrp, size_t padding,
 910     cred_t *fcred)
 911 {
 912         int error = 0;
 913         struct as *as = curproc->p_as;
 914         caddr_t addr = NULL;
 915         caddr_t start_addr;
 916         size_t len;
 917         size_t pad_len;
 918         int prot = PROT_USER | PROT_READ;
 919         uint_t ma_flags = _MAP_LOW32;
 920         vattr_t vattr;
 921         struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_USER, PROT_ALL);
 922 
 923         if (get_udatamodel() == DATAMODEL_LP64) {
 924                 ma_flags = 0;
 925         }
 926 
 927         vattr.va_mask = AT_SIZE;
 928         error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
 929         if (error) {
 930                 return (error);
 931         }
 932 
 933         len = vattr.va_size;
 934 
 935         ma_flags |= MAP_PRIVATE;
 936         if (padding == 0) {
 937                 MOBJ_STAT_ADD(map_flat_no_padding);
 938                 error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL,
 939                     ma_flags, fcred, NULL);
 940                 if (error == 0) {
 941                         mrp[0].mr_addr = addr;
 942                         mrp[0].mr_msize = len;
 943                         mrp[0].mr_fsize = len;
 944                         mrp[0].mr_offset = 0;
 945                         mrp[0].mr_prot = prot;
 946                         mrp[0].mr_flags = 0;
 947                 }
 948                 return (error);
 949         }
 950 
 951         /* padding was requested so there's more work to be done */
 952         MOBJ_STAT_ADD(map_flat_padding);
 953 
 954         /* No need to reserve swap space now since it will be reserved later */
 955         crargs.flags |= MAP_NORESERVE;
 956 
 957         /* Need to setup padding which can only be in PAGESIZE increments. */
 958         ASSERT((padding & PAGEOFFSET) == 0);
 959         pad_len = len + (2 * padding);
 960 
 961         as_rangelock(as);
 962         map_addr(&addr, pad_len, 0, 1, ma_flags);
 963         error = as_map(as, addr, pad_len, segvn_create, &crargs);
 964         as_rangeunlock(as);
 965         if (error) {
 966                 return (error);
 967         }
 968         start_addr = addr;
 969         addr += padding;
 970         ma_flags |= MAP_FIXED;
 971         error = VOP_MAP(vp, 0, as, &addr, len, prot, PROT_ALL, ma_flags,
 972             fcred, NULL);
 973         if (error == 0) {
 974                 mrp[0].mr_addr = start_addr;
 975                 mrp[0].mr_msize = padding;
 976                 mrp[0].mr_fsize = 0;
 977                 mrp[0].mr_offset = 0;
 978                 mrp[0].mr_prot = 0;
 979                 mrp[0].mr_flags = MR_PADDING;
 980 
 981                 mrp[1].mr_addr = addr;
 982                 mrp[1].mr_msize = len;
 983                 mrp[1].mr_fsize = len;
 984                 mrp[1].mr_offset = 0;
 985                 mrp[1].mr_prot = prot;
 986                 mrp[1].mr_flags = 0;
 987 
 988                 mrp[2].mr_addr = addr + P2ROUNDUP(len, PAGESIZE);
 989                 mrp[2].mr_msize = padding;
 990                 mrp[2].mr_fsize = 0;
 991                 mrp[2].mr_offset = 0;
 992                 mrp[2].mr_prot = 0;
 993                 mrp[2].mr_flags = MR_PADDING;
 994         } else {
 995                 /* Need to cleanup the as_map from earlier */
 996                 (void) as_unmap(as, start_addr, pad_len);
 997         }
 998         return (error);
 999 }
1000 
1001 /*
1002  * Map a PT_LOAD or PT_SUNWBSS section of an executable file into the user's
1003  * address space.
1004  * vp - vnode to be mapped in
1005  * addr - start address
1006  * len - length of vp to be mapped
1007  * zfodlen - length of zero filled memory after len above
1008  * offset - offset into file where mapping should start
1009  * prot - protections for this mapping
1010  * fcred - credentials for the file associated with vp at open time.
1011  */
1012 static int
1013 mmapobj_map_ptload(struct vnode *vp, caddr_t addr, size_t len, size_t zfodlen,
1014     off_t offset, int prot, cred_t *fcred)
1015 {
1016         int error = 0;
1017         caddr_t zfodbase, oldaddr;
1018         size_t oldlen;
1019         size_t end;
1020         size_t zfoddiff;
1021         label_t ljb;
1022         struct as *as = curproc->p_as;
1023         model_t model;
1024         int full_page;
1025 
1026         /*
1027          * See if addr and offset are aligned such that we can map in
1028          * full pages instead of partial pages.
1029          */
1030         full_page = (((uintptr_t)addr & PAGEOFFSET) ==
1031             ((uintptr_t)offset & PAGEOFFSET));
1032 
1033         model = get_udatamodel();
1034 
1035         oldaddr = addr;
1036         addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1037         if (len) {
1038                 spgcnt_t availm, npages;
1039                 int preread;
1040                 uint_t mflag = MAP_PRIVATE | MAP_FIXED;
1041 
1042                 if (model == DATAMODEL_ILP32) {
1043                         mflag |= _MAP_LOW32;
1044                 }
1045                 /* We may need to map in extra bytes */
1046                 oldlen = len;
1047                 len += ((size_t)oldaddr & PAGEOFFSET);
1048 
1049                 if (full_page) {
1050                         offset = (off_t)((uintptr_t)offset & PAGEMASK);
1051                         if ((prot & (PROT_WRITE | PROT_EXEC)) == PROT_EXEC) {
1052                                 mflag |= MAP_TEXT;
1053                                 MOBJ_STAT_ADD(map_ptload_text);
1054                         } else {
1055                                 mflag |= MAP_INITDATA;
1056                                 MOBJ_STAT_ADD(map_ptload_initdata);
1057                         }
1058 
1059                         /*
1060                          * maxprot is passed as PROT_ALL so that mdb can
1061                          * write to this segment.
1062                          */
1063                         if (error = VOP_MAP(vp, (offset_t)offset, as, &addr,
1064                             len, prot, PROT_ALL, mflag, fcred, NULL)) {
1065                                 return (error);
1066                         }
1067 
1068                         /*
1069                          * If the segment can fit and is relatively small, then
1070                          * we prefault the entire segment in.  This is based
1071                          * on the model that says the best working set of a
1072                          * small program is all of its pages.
1073                          * We only do this if freemem will not drop below
1074                          * lotsfree since we don't want to induce paging.
1075                          */
1076                         npages = (spgcnt_t)btopr(len);
1077                         availm = freemem - lotsfree;
1078                         preread = (npages < availm && len < PGTHRESH) ? 1 : 0;
1079 
1080                         /*
1081                          * If we aren't prefaulting the segment,
1082                          * increment "deficit", if necessary to ensure
1083                          * that pages will become available when this
1084                          * process starts executing.
1085                          */
1086                         if (preread == 0 && npages > availm &&
1087                             deficit < lotsfree) {
1088                                 deficit += MIN((pgcnt_t)(npages - availm),
1089                                     lotsfree - deficit);
1090                         }
1091 
1092                         if (preread) {
1093                                 (void) as_faulta(as, addr, len);
1094                                 MOBJ_STAT_ADD(map_ptload_preread);
1095                         }
1096                 } else {
1097                         /*
1098                          * addr and offset were not aligned such that we could
1099                          * use VOP_MAP, thus we need to as_map the memory we
1100                          * need and then read the data in from disk.
1101                          * This code path is a corner case which should never
1102                          * be taken, but hand crafted binaries could trigger
1103                          * this logic and it needs to work correctly.
1104                          */
1105                         MOBJ_STAT_ADD(map_ptload_unaligned_text);
1106                         as_rangelock(as);
1107                         (void) as_unmap(as, addr, len);
1108 
1109                         /*
1110                          * We use zfod_argsp because we need to be able to
1111                          * write to the mapping and then we'll change the
1112                          * protections later if they are incorrect.
1113                          */
1114                         error = as_map(as, addr, len, segvn_create, zfod_argsp);
1115                         as_rangeunlock(as);
1116                         if (error) {
1117                                 MOBJ_STAT_ADD(map_ptload_unaligned_map_fail);
1118                                 return (error);
1119                         }
1120 
1121                         /* Now read in the data from disk */
1122                         error = vn_rdwr(UIO_READ, vp, oldaddr, oldlen, offset,
1123                             UIO_USERSPACE, 0, (rlim64_t)0, fcred, NULL);
1124                         if (error) {
1125                                 MOBJ_STAT_ADD(map_ptload_unaligned_read_fail);
1126                                 return (error);
1127                         }
1128 
1129                         /*
1130                          * Now set protections.
1131                          */
1132                         if (prot != PROT_ZFOD) {
1133                                 (void) as_setprot(as, addr, len, prot);
1134                         }
1135                 }
1136         }
1137 
1138         if (zfodlen) {
1139                 end = (size_t)addr + len;
1140                 zfodbase = (caddr_t)P2ROUNDUP(end, PAGESIZE);
1141                 zfoddiff = (uintptr_t)zfodbase - end;
1142                 if (zfoddiff) {
1143                         /*
1144                          * Before we go to zero the remaining space on the last
1145                          * page, make sure we have write permission.
1146                          *
1147                          * We need to be careful how we zero-fill the last page
1148                          * if the protection does not include PROT_WRITE. Using
1149                          * as_setprot() can cause the VM segment code to call
1150                          * segvn_vpage(), which must allocate a page struct for
1151                          * each page in the segment. If we have a very large
1152                          * segment, this may fail, so we check for that, even
1153                          * though we ignore other return values from as_setprot.
1154                          */
1155                         MOBJ_STAT_ADD(zfoddiff);
1156                         if ((prot & PROT_WRITE) == 0) {
1157                                 if (as_setprot(as, (caddr_t)end, zfoddiff,
1158                                     prot | PROT_WRITE) == ENOMEM)
1159                                         return (ENOMEM);
1160                                 MOBJ_STAT_ADD(zfoddiff_nowrite);
1161                         }
1162                         if (on_fault(&ljb)) {
1163                                 no_fault();
1164                                 if ((prot & PROT_WRITE) == 0) {
1165                                         (void) as_setprot(as, (caddr_t)end,
1166                                             zfoddiff, prot);
1167                                 }
1168                                 return (EFAULT);
1169                         }
1170                         uzero((void *)end, zfoddiff);
1171                         no_fault();
1172 
1173                         /*
1174                          * Remove write protection to return to original state
1175                          */
1176                         if ((prot & PROT_WRITE) == 0) {
1177                                 (void) as_setprot(as, (caddr_t)end,
1178                                     zfoddiff, prot);
1179                         }
1180                 }
1181                 if (zfodlen > zfoddiff) {
1182                         struct segvn_crargs crargs =
1183                             SEGVN_ZFOD_ARGS(prot, PROT_ALL);
1184 
1185                         MOBJ_STAT_ADD(zfodextra);
1186                         zfodlen -= zfoddiff;
1187                         crargs.szc = AS_MAP_NO_LPOOB;
1188 
1189 
1190                         as_rangelock(as);
1191                         (void) as_unmap(as, (caddr_t)zfodbase, zfodlen);
1192                         error = as_map(as, (caddr_t)zfodbase,
1193                             zfodlen, segvn_create, &crargs);
1194                         as_rangeunlock(as);
1195                         if (error) {
1196                                 return (error);
1197                         }
1198                 }
1199         }
1200         return (0);
1201 }
1202 
1203 /*
1204  * Map the ELF file represented by vp into the users address space.  The
1205  * first mapping will start at start_addr and there will be num_elements
1206  * mappings.  The mappings are described by the data in mrp which may be
1207  * modified upon returning from this function.
1208  * Returns 0 for success or errno for failure.
1209  */
1210 static int
1211 mmapobj_map_elf(struct vnode *vp, caddr_t start_addr, mmapobj_result_t *mrp,
1212     int num_elements, cred_t *fcred, ushort_t e_type)
1213 {
1214         int i;
1215         int ret;
1216         caddr_t lo;
1217         caddr_t hi;
1218         struct as *as = curproc->p_as;
1219 
1220         for (i = 0; i < num_elements; i++) {
1221                 caddr_t addr;
1222                 size_t p_memsz;
1223                 size_t p_filesz;
1224                 size_t zfodlen;
1225                 offset_t p_offset;
1226                 size_t dif;
1227                 int prot;
1228 
1229                 /* Always need to adjust mr_addr */
1230                 addr = start_addr + (size_t)(mrp[i].mr_addr);
1231                 mrp[i].mr_addr =
1232                     (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1233 
1234                 /* Padding has already been mapped */
1235                 if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
1236                         continue;
1237                 }
1238                 p_memsz = mrp[i].mr_msize;
1239                 p_filesz = mrp[i].mr_fsize;
1240                 zfodlen = p_memsz - p_filesz;
1241                 p_offset = mrp[i].mr_offset;
1242                 dif = (uintptr_t)(addr) & PAGEOFFSET;
1243                 prot = mrp[i].mr_prot | PROT_USER;
1244                 ret = mmapobj_map_ptload(vp, addr, p_filesz, zfodlen,
1245                     p_offset, prot, fcred);
1246                 if (ret != 0) {
1247                         MOBJ_STAT_ADD(ptload_failed);
1248                         mmapobj_unmap(mrp, i, num_elements, e_type);
1249                         return (ret);
1250                 }
1251 
1252                 /* Need to cleanup mrp to reflect the actual values used */
1253                 mrp[i].mr_msize += dif;
1254                 mrp[i].mr_offset = (size_t)addr & PAGEOFFSET;
1255         }
1256 
1257         /* Also need to unmap any holes created above */
1258         if (num_elements == 1) {
1259                 MOBJ_STAT_ADD(map_elf_no_holes);
1260                 return (0);
1261         }
1262         if (e_type == ET_EXEC) {
1263                 return (0);
1264         }
1265 
1266         as_rangelock(as);
1267         lo = start_addr;
1268         hi = mrp[0].mr_addr;
1269 
1270         /* Remove holes made by the rest of the segments */
1271         for (i = 0; i < num_elements - 1; i++) {
1272                 lo = (caddr_t)P2ROUNDUP((size_t)(mrp[i].mr_addr) +
1273                     mrp[i].mr_msize, PAGESIZE);
1274                 hi = mrp[i + 1].mr_addr;
1275                 if (lo < hi) {
1276                         /*
1277                          * If as_unmap fails we just use up a bit of extra
1278                          * space
1279                          */
1280                         (void) as_unmap(as, (caddr_t)lo,
1281                             (size_t)hi - (size_t)lo);
1282                         MOBJ_STAT_ADD(unmap_hole);
1283                 }
1284         }
1285         as_rangeunlock(as);
1286 
1287         return (0);
1288 }
1289 
1290 /* Ugly hack to get STRUCT_* macros to work below */
1291 struct myphdr {
1292         Phdr            x;      /* native version */
1293 };
1294 
1295 struct myphdr32 {
1296         Elf32_Phdr      x;
1297 };
1298 
1299 /*
1300  * Calculate and return the number of loadable segments in the ELF Phdr
1301  * represented by phdrbase as well as the len of the total mapping and
1302  * the max alignment that is needed for a given segment.  On success,
1303  * 0 is returned, and *len, *loadable and *align have been filled out.
1304  * On failure, errno will be returned, which in this case is ENOTSUP
1305  * if we were passed an ELF file with overlapping segments.
1306  */
1307 static int
1308 calc_loadable(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, size_t *len,
1309     int *loadable, size_t *align)
1310 {
1311         int i;
1312         int hsize;
1313         model_t model;
1314         ushort_t e_type = ehdrp->e_type;     /* same offset 32 and 64 bit */
1315         uint_t p_type;
1316         offset_t p_offset;
1317         size_t p_memsz;
1318         size_t p_align;
1319         caddr_t vaddr;
1320         int num_segs = 0;
1321         caddr_t start_addr = NULL;
1322         caddr_t p_end = NULL;
1323         size_t max_align = 0;
1324         size_t min_align = PAGESIZE;    /* needed for vmem_xalloc */
1325         STRUCT_HANDLE(myphdr, mph);
1326 #if defined(__sparc)
1327         extern int vac_size;
1328 
1329         /*
1330          * Want to prevent aliasing by making the start address at least be
1331          * aligned to vac_size.
1332          */
1333         min_align = MAX(PAGESIZE, vac_size);
1334 #endif
1335 
1336         model = get_udatamodel();
1337         STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1338 
1339         /* hsize alignment should have been checked before calling this func */
1340         if (model == DATAMODEL_LP64) {
1341                 hsize = ehdrp->e_phentsize;
1342                 if (hsize & 7) {
1343                         return (ENOTSUP);
1344                 }
1345         } else {
1346                 ASSERT(model == DATAMODEL_ILP32);
1347                 hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
1348                 if (hsize & 3) {
1349                         return (ENOTSUP);
1350                 }
1351         }
1352 
1353         /*
1354          * Determine the span of all loadable segments and calculate the
1355          * number of loadable segments.
1356          */
1357         for (i = 0; i < nphdrs; i++) {
1358                 p_type = STRUCT_FGET(mph, x.p_type);
1359                 if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
1360                         vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
1361                         p_memsz = STRUCT_FGET(mph, x.p_memsz);
1362 
1363                         /*
1364                          * Skip this header if it requests no memory to be
1365                          * mapped.
1366                          */
1367                         if (p_memsz == 0) {
1368                                 STRUCT_SET_HANDLE(mph, model,
1369                                     (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1370                                     hsize));
1371                                 MOBJ_STAT_ADD(nomem_header);
1372                                 continue;
1373                         }
1374                         if (num_segs++ == 0) {
1375                                 /*
1376                                  * The p_vaddr of the first PT_LOAD segment
1377                                  * must either be NULL or within the first
1378                                  * page in order to be interpreted.
1379                                  * Otherwise, its an invalid file.
1380                                  */
1381                                 if (e_type == ET_DYN &&
1382                                     ((caddr_t)((uintptr_t)vaddr &
1383                                     (uintptr_t)PAGEMASK) != NULL)) {
1384                                         MOBJ_STAT_ADD(inval_header);
1385                                         return (ENOTSUP);
1386                                 }
1387                                 start_addr = vaddr;
1388                                 /*
1389                                  * For the first segment, we need to map from
1390                                  * the beginning of the file, so we will
1391                                  * adjust the size of the mapping to include
1392                                  * this memory.
1393                                  */
1394                                 p_offset = STRUCT_FGET(mph, x.p_offset);
1395                         } else {
1396                                 p_offset = 0;
1397                         }
1398                         /*
1399                          * Check to make sure that this mapping wouldn't
1400                          * overlap a previous mapping.
1401                          */
1402                         if (vaddr < p_end) {
1403                                 MOBJ_STAT_ADD(overlap_header);
1404                                 return (ENOTSUP);
1405                         }
1406 
1407                         p_end = vaddr + p_memsz + p_offset;
1408                         p_end = (caddr_t)P2ROUNDUP((size_t)p_end, PAGESIZE);
1409 
1410                         p_align = STRUCT_FGET(mph, x.p_align);
1411                         if (p_align > 1 && p_align > max_align) {
1412                                 max_align = p_align;
1413                                 if (max_align < min_align) {
1414                                         max_align = min_align;
1415                                         MOBJ_STAT_ADD(min_align);
1416                                 }
1417                         }
1418                 }
1419                 STRUCT_SET_HANDLE(mph, model,
1420                     (struct myphdr *)((size_t)STRUCT_BUF(mph) + hsize));
1421         }
1422 
1423         /*
1424          * The alignment should be a power of 2, if it isn't we forgive it
1425          * and round up.  On overflow, we'll set the alignment to max_align
1426          * rounded down to the nearest power of 2.
1427          */
1428         if (max_align > 0 && !ISP2(max_align)) {
1429                 MOBJ_STAT_ADD(np2_align);
1430                 *align = 2 * (1L << (highbit(max_align) - 1));
1431                 if (*align < max_align ||
1432                     (*align > UINT_MAX && model == DATAMODEL_ILP32)) {
1433                         MOBJ_STAT_ADD(np2_align_overflow);
1434                         *align = 1L << (highbit(max_align) - 1);
1435                 }
1436         } else {
1437                 *align = max_align;
1438         }
1439 
1440         ASSERT(*align >= PAGESIZE || *align == 0);
1441 
1442         *loadable = num_segs;
1443         *len = p_end - start_addr;
1444         return (0);
1445 }
1446 
1447 /*
1448  * Check the address space to see if the virtual addresses to be used are
1449  * available.  If they are not, return errno for failure.  On success, 0
1450  * will be returned, and the virtual addresses for each mmapobj_result_t
1451  * will be reserved.  Note that a reservation could have earlier been made
1452  * for a given segment via a /dev/null mapping.  If that is the case, then
1453  * we can use that VA space for our mappings.
1454  * Note: this function will only be used for ET_EXEC binaries.
1455  */
1456 int
1457 check_exec_addrs(int loadable, mmapobj_result_t *mrp, caddr_t start_addr)
1458 {
1459         int i;
1460         struct as *as = curproc->p_as;
1461         struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
1462         int ret;
1463         caddr_t myaddr;
1464         size_t mylen;
1465         struct seg *seg;
1466 
1467         /* No need to reserve swap space now since it will be reserved later */
1468         crargs.flags |= MAP_NORESERVE;
1469         as_rangelock(as);
1470         for (i = 0; i < loadable; i++) {
1471 
1472                 myaddr = start_addr + (size_t)mrp[i].mr_addr;
1473                 mylen = mrp[i].mr_msize;
1474 
1475                 /* See if there is a hole in the as for this range */
1476                 if (as_gap(as, mylen, &myaddr, &mylen, 0, NULL) == 0) {
1477                         ASSERT(myaddr == start_addr + (size_t)mrp[i].mr_addr);
1478                         ASSERT(mylen == mrp[i].mr_msize);
1479 
1480 #ifdef DEBUG
1481                         if (MR_GET_TYPE(mrp[i].mr_flags) == MR_PADDING) {
1482                                 MOBJ_STAT_ADD(exec_padding);
1483                         }
1484 #endif
1485                         ret = as_map(as, myaddr, mylen, segvn_create, &crargs);
1486                         if (ret) {
1487                                 as_rangeunlock(as);
1488                                 mmapobj_unmap_exec(mrp, i, start_addr);
1489                                 return (ret);
1490                         }
1491                 } else {
1492                         /*
1493                          * There is a mapping that exists in the range
1494                          * so check to see if it was a "reservation"
1495                          * from /dev/null.  The mapping is from
1496                          * /dev/null if the mapping comes from
1497                          * segdev and the type is neither MAP_SHARED
1498                          * nor MAP_PRIVATE.
1499                          */
1500                         AS_LOCK_ENTER(as, RW_READER);
1501                         seg = as_findseg(as, myaddr, 0);
1502                         MOBJ_STAT_ADD(exec_addr_mapped);
1503                         if (seg && seg->s_ops == &segdev_ops &&
1504                             ((SEGOP_GETTYPE(seg, myaddr) &
1505                             (MAP_SHARED | MAP_PRIVATE)) == 0) &&
1506                             myaddr >= seg->s_base &&
1507                             myaddr + mylen <=
1508                             seg->s_base + seg->s_size) {
1509                                 MOBJ_STAT_ADD(exec_addr_devnull);
1510                                 AS_LOCK_EXIT(as);
1511                                 (void) as_unmap(as, myaddr, mylen);
1512                                 ret = as_map(as, myaddr, mylen, segvn_create,
1513                                     &crargs);
1514                                 mrp[i].mr_flags |= MR_RESV;
1515                                 if (ret) {
1516                                         as_rangeunlock(as);
1517                                         /* Need to remap what we unmapped */
1518                                         mmapobj_unmap_exec(mrp, i + 1,
1519                                             start_addr);
1520                                         return (ret);
1521                                 }
1522                         } else {
1523                                 AS_LOCK_EXIT(as);
1524                                 as_rangeunlock(as);
1525                                 mmapobj_unmap_exec(mrp, i, start_addr);
1526                                 MOBJ_STAT_ADD(exec_addr_in_use);
1527                                 return (EADDRINUSE);
1528                         }
1529                 }
1530         }
1531         as_rangeunlock(as);
1532         return (0);
1533 }
1534 
1535 /*
1536  * Walk through the ELF program headers and extract all useful information
1537  * for PT_LOAD and PT_SUNWBSS segments into mrp.
1538  * Return 0 on success or error on failure.
1539  */
1540 static int
1541 process_phdrs(Ehdr *ehdrp, caddr_t phdrbase, int nphdrs, mmapobj_result_t *mrp,
1542     vnode_t *vp, uint_t *num_mapped, size_t padding, cred_t *fcred)
1543 {
1544         int i;
1545         caddr_t start_addr = NULL;
1546         caddr_t vaddr;
1547         size_t len = 0;
1548         size_t lib_len = 0;
1549         int ret;
1550         int prot;
1551         struct lib_va *lvp = NULL;
1552         vattr_t vattr;
1553         struct as *as = curproc->p_as;
1554         int error;
1555         int loadable = 0;
1556         int current = 0;
1557         int use_lib_va = 1;
1558         size_t align = 0;
1559         size_t add_pad = 0;
1560         int hdr_seen = 0;
1561         ushort_t e_type = ehdrp->e_type;     /* same offset 32 and 64 bit */
1562         uint_t p_type;
1563         offset_t p_offset;
1564         size_t p_memsz;
1565         size_t p_filesz;
1566         uint_t p_flags;
1567         int hsize;
1568         model_t model;
1569         STRUCT_HANDLE(myphdr, mph);
1570 
1571         model = get_udatamodel();
1572         STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1573 
1574         /*
1575          * Need to make sure that hsize is aligned properly.
1576          * For 32bit processes, 4 byte alignment is required.
1577          * For 64bit processes, 8 byte alignment is required.
1578          * If the alignment isn't correct, we need to return failure
1579          * since it could cause an alignment error panic while walking
1580          * the phdr array.
1581          */
1582         if (model == DATAMODEL_LP64) {
1583                 hsize = ehdrp->e_phentsize;
1584                 if (hsize & 7) {
1585                         MOBJ_STAT_ADD(phent_align64);
1586                         return (ENOTSUP);
1587                 }
1588         } else {
1589                 ASSERT(model == DATAMODEL_ILP32);
1590                 hsize = ((Elf32_Ehdr *)ehdrp)->e_phentsize;
1591                 if (hsize & 3) {
1592                         MOBJ_STAT_ADD(phent_align32);
1593                         return (ENOTSUP);
1594                 }
1595         }
1596 
1597         if ((padding != 0) || secflag_enabled(curproc, PROC_SEC_ASLR)) {
1598                 use_lib_va = 0;
1599         }
1600         if (e_type == ET_DYN) {
1601                 vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME;
1602                 error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
1603                 if (error) {
1604                         return (error);
1605                 }
1606                 /* Check to see if we already have a description for this lib */
1607                 if (!secflag_enabled(curproc, PROC_SEC_ASLR))
1608                         lvp = lib_va_find(&vattr);
1609 
1610                 if (lvp != NULL) {
1611                         MOBJ_STAT_ADD(lvp_found);
1612                         if (use_lib_va) {
1613                                 start_addr = mmapobj_lookup_start_addr(lvp);
1614                                 if (start_addr == NULL) {
1615                                         lib_va_release(lvp);
1616                                         return (ENOMEM);
1617                                 }
1618                         }
1619 
1620                         /*
1621                          * loadable may be zero if the original allocator
1622                          * of lvp hasn't finished setting it up but the rest
1623                          * of the fields will be accurate.
1624                          */
1625                         loadable = lvp->lv_num_segs;
1626                         len = lvp->lv_len;
1627                         align = lvp->lv_align;
1628                 }
1629         }
1630 
1631         /*
1632          * Determine the span of all loadable segments and calculate the
1633          * number of loadable segments, the total len spanned by the mappings
1634          * and the max alignment, if we didn't get them above.
1635          */
1636         if (loadable == 0) {
1637                 MOBJ_STAT_ADD(no_loadable_yet);
1638                 ret = calc_loadable(ehdrp, phdrbase, nphdrs, &len,
1639                     &loadable, &align);
1640                 if (ret != 0) {
1641                         /*
1642                          * Since it'd be an invalid file, we shouldn't have
1643                          * cached it previously.
1644                          */
1645                         ASSERT(lvp == NULL);
1646                         return (ret);
1647                 }
1648 #ifdef DEBUG
1649                 if (lvp) {
1650                         ASSERT(len == lvp->lv_len);
1651                         ASSERT(align == lvp->lv_align);
1652                 }
1653 #endif
1654         }
1655 
1656         /* Make sure there's something to map. */
1657         if (len == 0 || loadable == 0) {
1658                 /*
1659                  * Since it'd be an invalid file, we shouldn't have
1660                  * cached it previously.
1661                  */
1662                 ASSERT(lvp == NULL);
1663                 MOBJ_STAT_ADD(nothing_to_map);
1664                 return (ENOTSUP);
1665         }
1666 
1667         lib_len = len;
1668         if (padding != 0) {
1669                 loadable += 2;
1670         }
1671         if (loadable > *num_mapped) {
1672                 *num_mapped = loadable;
1673                 /* cleanup previous reservation */
1674                 if (start_addr) {
1675                         (void) as_unmap(as, start_addr, lib_len);
1676                 }
1677                 MOBJ_STAT_ADD(e2big);
1678                 if (lvp) {
1679                         lib_va_release(lvp);
1680                 }
1681                 return (E2BIG);
1682         }
1683 
1684         /*
1685          * We now know the size of the object to map and now we need to
1686          * get the start address to map it at.  It's possible we already
1687          * have it if we found all the info we need in the lib_va cache.
1688          */
1689         if (e_type == ET_DYN && start_addr == NULL) {
1690                 /*
1691                  * Need to make sure padding does not throw off
1692                  * required alignment.  We can only specify an
1693                  * alignment for the starting address to be mapped,
1694                  * so we round padding up to the alignment and map
1695                  * from there and then throw out the extra later.
1696                  */
1697                 if (padding != 0) {
1698                         if (align > 1) {
1699                                 add_pad = P2ROUNDUP(padding, align);
1700                                 len += add_pad;
1701                                 MOBJ_STAT_ADD(dyn_pad_align);
1702                         } else {
1703                                 MOBJ_STAT_ADD(dyn_pad_noalign);
1704                                 len += padding; /* at beginning */
1705                         }
1706                         len += padding; /* at end of mapping */
1707                 }
1708                 /*
1709                  * At this point, if lvp is non-NULL, then above we
1710                  * already found it in the cache but did not get
1711                  * the start address since we were not going to use lib_va.
1712                  * Since we know that lib_va will not be used, it's safe
1713                  * to call mmapobj_alloc_start_addr and know that lvp
1714                  * will not be modified.
1715                  */
1716                 ASSERT(lvp ? use_lib_va == 0 : 1);
1717                 start_addr = mmapobj_alloc_start_addr(&lvp, len,
1718                     use_lib_va,
1719                     secflag_enabled(curproc, PROC_SEC_ASLR),
1720                     align, &vattr);
1721                 if (start_addr == NULL) {
1722                         if (lvp) {
1723                                 lib_va_release(lvp);
1724                         }
1725                         MOBJ_STAT_ADD(alloc_start_fail);
1726                         return (ENOMEM);
1727                 }
1728                 /*
1729                  * If we can't cache it, no need to hang on to it.
1730                  * Setting lv_num_segs to non-zero will make that
1731                  * field active and since there are too many segments
1732                  * to cache, all future users will not try to use lv_mps.
1733                  */
1734                 if (lvp != NULL && loadable > LIBVA_CACHED_SEGS && use_lib_va) {
1735                         lvp->lv_num_segs = loadable;
1736                         lib_va_release(lvp);
1737                         lvp = NULL;
1738                         MOBJ_STAT_ADD(lvp_nocache);
1739                 }
1740                 /*
1741                  * Free the beginning of the mapping if the padding
1742                  * was not aligned correctly.
1743                  */
1744                 if (padding != 0 && add_pad != padding) {
1745                         (void) as_unmap(as, start_addr,
1746                             add_pad - padding);
1747                         start_addr += (add_pad - padding);
1748                         MOBJ_STAT_ADD(extra_padding);
1749                 }
1750         }
1751 
1752         /*
1753          * At this point, we have reserved the virtual address space
1754          * for our mappings.  Now we need to start filling out the mrp
1755          * array to describe all of the individual mappings we are going
1756          * to return.
1757          * For ET_EXEC there has been no memory reservation since we are
1758          * using fixed addresses.  While filling in the mrp array below,
1759          * we will have the first segment biased to start at addr 0
1760          * and the rest will be biased by this same amount.  Thus if there
1761          * is padding, the first padding will start at addr 0, and the next
1762          * segment will start at the value of padding.
1763          */
1764 
1765         /* We'll fill out padding later, so start filling in mrp at index 1 */
1766         if (padding != 0) {
1767                 current = 1;
1768         }
1769 
1770         /* If we have no more need for lvp let it go now */
1771         if (lvp != NULL && use_lib_va == 0) {
1772                 lib_va_release(lvp);
1773                 MOBJ_STAT_ADD(lvp_not_needed);
1774                 lvp = NULL;
1775         }
1776 
1777         /* Now fill out the mrp structs from the program headers */
1778         STRUCT_SET_HANDLE(mph, model, (struct myphdr *)phdrbase);
1779         for (i = 0; i < nphdrs; i++) {
1780                 p_type = STRUCT_FGET(mph, x.p_type);
1781                 if (p_type == PT_LOAD || p_type == PT_SUNWBSS) {
1782                         vaddr = (caddr_t)(uintptr_t)STRUCT_FGET(mph, x.p_vaddr);
1783                         p_memsz = STRUCT_FGET(mph, x.p_memsz);
1784                         p_filesz = STRUCT_FGET(mph, x.p_filesz);
1785                         p_offset = STRUCT_FGET(mph, x.p_offset);
1786                         p_flags = STRUCT_FGET(mph, x.p_flags);
1787 
1788                         /*
1789                          * Skip this header if it requests no memory to be
1790                          * mapped.
1791                          */
1792                         if (p_memsz == 0) {
1793                                 STRUCT_SET_HANDLE(mph, model,
1794                                     (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1795                                     hsize));
1796                                 MOBJ_STAT_ADD(no_mem_map_sz);
1797                                 continue;
1798                         }
1799 
1800                         prot = 0;
1801                         if (p_flags & PF_R)
1802                                 prot |= PROT_READ;
1803                         if (p_flags & PF_W)
1804                                 prot |= PROT_WRITE;
1805                         if (p_flags & PF_X)
1806                                 prot |= PROT_EXEC;
1807 
1808                         ASSERT(current < loadable);
1809                         mrp[current].mr_msize = p_memsz;
1810                         mrp[current].mr_fsize = p_filesz;
1811                         mrp[current].mr_offset = p_offset;
1812                         mrp[current].mr_prot = prot;
1813 
1814                         if (hdr_seen == 0 && p_filesz != 0) {
1815                                 mrp[current].mr_flags = MR_HDR_ELF;
1816                                 /*
1817                                  * We modify mr_offset because we
1818                                  * need to map the ELF header as well, and if
1819                                  * we didn't then the header could be left out
1820                                  * of the mapping that we will create later.
1821                                  * Since we're removing the offset, we need to
1822                                  * account for that in the other fields as well
1823                                  * since we will be mapping the memory from 0
1824                                  * to p_offset.
1825                                  */
1826                                 if (e_type == ET_DYN) {
1827                                         mrp[current].mr_offset = 0;
1828                                         mrp[current].mr_msize += p_offset;
1829                                         mrp[current].mr_fsize += p_offset;
1830                                 } else {
1831                                         ASSERT(e_type == ET_EXEC);
1832                                         /*
1833                                          * Save off the start addr which will be
1834                                          * our bias for the rest of the
1835                                          * ET_EXEC mappings.
1836                                          */
1837                                         start_addr = vaddr - padding;
1838                                 }
1839                                 mrp[current].mr_addr = (caddr_t)padding;
1840                                 hdr_seen = 1;
1841                         } else {
1842                                 if (e_type == ET_EXEC) {
1843                                         /* bias mr_addr */
1844                                         mrp[current].mr_addr =
1845                                             vaddr - (size_t)start_addr;
1846                                 } else {
1847                                         mrp[current].mr_addr = vaddr + padding;
1848                                 }
1849                                 mrp[current].mr_flags = 0;
1850                         }
1851                         current++;
1852                 }
1853 
1854                 /* Move to next phdr */
1855                 STRUCT_SET_HANDLE(mph, model,
1856                     (struct myphdr *)((size_t)STRUCT_BUF(mph) +
1857                     hsize));
1858         }
1859 
1860         /* Now fill out the padding segments */
1861         if (padding != 0) {
1862                 mrp[0].mr_addr = NULL;
1863                 mrp[0].mr_msize = padding;
1864                 mrp[0].mr_fsize = 0;
1865                 mrp[0].mr_offset = 0;
1866                 mrp[0].mr_prot = 0;
1867                 mrp[0].mr_flags = MR_PADDING;
1868 
1869                 /* Setup padding for the last segment */
1870                 ASSERT(current == loadable - 1);
1871                 mrp[current].mr_addr = (caddr_t)lib_len + padding;
1872                 mrp[current].mr_msize = padding;
1873                 mrp[current].mr_fsize = 0;
1874                 mrp[current].mr_offset = 0;
1875                 mrp[current].mr_prot = 0;
1876                 mrp[current].mr_flags = MR_PADDING;
1877         }
1878 
1879         /*
1880          * Need to make sure address ranges desired are not in use or
1881          * are previously allocated reservations from /dev/null.  For
1882          * ET_DYN, we already made sure our address range was free.
1883          */
1884         if (e_type == ET_EXEC) {
1885                 ret = check_exec_addrs(loadable, mrp, start_addr);
1886                 if (ret != 0) {
1887                         ASSERT(lvp == NULL);
1888                         MOBJ_STAT_ADD(check_exec_failed);
1889                         return (ret);
1890                 }
1891         }
1892 
1893         /* Finish up our business with lvp. */
1894         if (lvp) {
1895                 ASSERT(e_type == ET_DYN);
1896                 if (lvp->lv_num_segs == 0 && loadable <= LIBVA_CACHED_SEGS) {
1897                         bcopy(mrp, lvp->lv_mps,
1898                             loadable * sizeof (mmapobj_result_t));
1899                         membar_producer();
1900                 }
1901                 /*
1902                  * Setting lv_num_segs to a non-zero value indicates that
1903                  * lv_mps is now valid and can be used by other threads.
1904                  * So, the above stores need to finish before lv_num_segs
1905                  * is updated. lv_mps is only valid if lv_num_segs is
1906                  * greater than LIBVA_CACHED_SEGS.
1907                  */
1908                 lvp->lv_num_segs = loadable;
1909                 lib_va_release(lvp);
1910                 MOBJ_STAT_ADD(lvp_used);
1911         }
1912 
1913         /* Now that we have mrp completely filled out go map it */
1914         ret = mmapobj_map_elf(vp, start_addr, mrp, loadable, fcred, e_type);
1915         if (ret == 0) {
1916                 *num_mapped = loadable;
1917         }
1918 
1919         return (ret);
1920 }
1921 
1922 /*
1923  * Take the ELF file passed in, and do the work of mapping it.
1924  * num_mapped in - # elements in user buffer
1925  * num_mapped out - # sections mapped and length of mrp array if
1926  *                      no errors.
1927  */
1928 static int
1929 doelfwork(Ehdr *ehdrp, vnode_t *vp, mmapobj_result_t *mrp,
1930     uint_t *num_mapped, size_t padding, cred_t *fcred)
1931 {
1932         int error;
1933         offset_t phoff;
1934         int nphdrs;
1935         unsigned char ei_class;
1936         unsigned short phentsize;
1937         ssize_t phsizep;
1938         caddr_t phbasep;
1939         int to_map;
1940         model_t model;
1941 
1942         ei_class = ehdrp->e_ident[EI_CLASS];
1943         model = get_udatamodel();
1944         if ((model == DATAMODEL_ILP32 && ei_class == ELFCLASS64) ||
1945             (model == DATAMODEL_LP64 && ei_class == ELFCLASS32)) {
1946                 MOBJ_STAT_ADD(wrong_model);
1947                 return (ENOTSUP);
1948         }
1949 
1950         /* Can't execute code from "noexec" mounted filesystem. */
1951         if (ehdrp->e_type == ET_EXEC &&
1952             (vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) {
1953                 MOBJ_STAT_ADD(noexec_fs);
1954                 return (EACCES);
1955         }
1956 
1957         /*
1958          * Relocatable and core files are mapped as a single flat file
1959          * since no interpretation is done on them by mmapobj.
1960          */
1961         if (ehdrp->e_type == ET_REL || ehdrp->e_type == ET_CORE) {
1962                 to_map = padding ? 3 : 1;
1963                 if (*num_mapped < to_map) {
1964                         *num_mapped = to_map;
1965                         MOBJ_STAT_ADD(e2big_et_rel);
1966                         return (E2BIG);
1967                 }
1968                 error = mmapobj_map_flat(vp, mrp, padding, fcred);
1969                 if (error == 0) {
1970                         *num_mapped = to_map;
1971                         mrp[padding ? 1 : 0].mr_flags = MR_HDR_ELF;
1972                         MOBJ_STAT_ADD(et_rel_mapped);
1973                 }
1974                 return (error);
1975         }
1976 
1977         /* Check for an unknown ELF type */
1978         if (ehdrp->e_type != ET_EXEC && ehdrp->e_type != ET_DYN) {
1979                 MOBJ_STAT_ADD(unknown_elf_type);
1980                 return (ENOTSUP);
1981         }
1982 
1983         if (ei_class == ELFCLASS32) {
1984                 Elf32_Ehdr *e32hdr = (Elf32_Ehdr *)ehdrp;
1985                 ASSERT(model == DATAMODEL_ILP32);
1986                 nphdrs = e32hdr->e_phnum;
1987                 phentsize = e32hdr->e_phentsize;
1988                 if (phentsize < sizeof (Elf32_Phdr)) {
1989                         MOBJ_STAT_ADD(phent32_too_small);
1990                         return (ENOTSUP);
1991                 }
1992                 phoff = e32hdr->e_phoff;
1993         } else if (ei_class == ELFCLASS64) {
1994                 Elf64_Ehdr *e64hdr = (Elf64_Ehdr *)ehdrp;
1995                 ASSERT(model == DATAMODEL_LP64);
1996                 nphdrs = e64hdr->e_phnum;
1997                 phentsize = e64hdr->e_phentsize;
1998                 if (phentsize < sizeof (Elf64_Phdr)) {
1999                         MOBJ_STAT_ADD(phent64_too_small);
2000                         return (ENOTSUP);
2001                 }
2002                 phoff = e64hdr->e_phoff;
2003         } else {
2004                 /* fallthrough case for an invalid ELF class */
2005                 MOBJ_STAT_ADD(inval_elf_class);
2006                 return (ENOTSUP);
2007         }
2008 
2009         /*
2010          * nphdrs should only have this value for core files which are handled
2011          * above as a single mapping.  If other file types ever use this
2012          * sentinel, then we'll add the support needed to handle this here.
2013          */
2014         if (nphdrs == PN_XNUM) {
2015                 MOBJ_STAT_ADD(too_many_phdrs);
2016                 return (ENOTSUP);
2017         }
2018 
2019         phsizep = nphdrs * phentsize;
2020 
2021         if (phsizep == 0) {
2022                 MOBJ_STAT_ADD(no_phsize);
2023                 return (ENOTSUP);
2024         }
2025 
2026         /* Make sure we only wait for memory if it's a reasonable request */
2027         if (phsizep > mmapobj_alloc_threshold) {
2028                 MOBJ_STAT_ADD(phsize_large);
2029                 if ((phbasep = kmem_alloc(phsizep, KM_NOSLEEP)) == NULL) {
2030                         MOBJ_STAT_ADD(phsize_xtralarge);
2031                         return (ENOMEM);
2032                 }
2033         } else {
2034                 phbasep = kmem_alloc(phsizep, KM_SLEEP);
2035         }
2036 
2037         if ((error = vn_rdwr(UIO_READ, vp, phbasep, phsizep,
2038             (offset_t)phoff, UIO_SYSSPACE, 0, (rlim64_t)0,
2039             fcred, NULL)) != 0) {
2040                 kmem_free(phbasep, phsizep);
2041                 return (error);
2042         }
2043 
2044         /* Now process the phdr's */
2045         error = process_phdrs(ehdrp, phbasep, nphdrs, mrp, vp, num_mapped,
2046             padding, fcred);
2047         kmem_free(phbasep, phsizep);
2048         return (error);
2049 }
2050 
2051 #if defined(__sparc)
2052 /*
2053  * Hack to support 64 bit kernels running AOUT 4.x programs.
2054  * This is the sizeof (struct nlist) for a 32 bit kernel.
2055  * Since AOUT programs are 32 bit only, they will never use the 64 bit
2056  * sizeof (struct nlist) and thus creating a #define is the simplest
2057  * way around this since this is a format which is not being updated.
2058  * This will be used in the place of sizeof (struct nlist) below.
2059  */
2060 #define NLIST_SIZE      (0xC)
2061 
2062 static int
2063 doaoutwork(vnode_t *vp, mmapobj_result_t *mrp,
2064     uint_t *num_mapped, struct exec *hdr, cred_t *fcred)
2065 {
2066         int error;
2067         size_t size;
2068         size_t osize;
2069         size_t nsize;   /* nlist size */
2070         size_t msize;
2071         size_t zfoddiff;
2072         caddr_t addr;
2073         caddr_t start_addr;
2074         struct as *as = curproc->p_as;
2075         int prot = PROT_USER | PROT_READ | PROT_EXEC;
2076         uint_t mflag = MAP_PRIVATE | _MAP_LOW32;
2077         offset_t off = 0;
2078         int segnum = 0;
2079         uint_t to_map;
2080         int is_library = 0;
2081         struct segvn_crargs crargs = SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
2082 
2083         /* Only 32bit apps supported by this file format */
2084         if (get_udatamodel() != DATAMODEL_ILP32) {
2085                 MOBJ_STAT_ADD(aout_64bit_try);
2086                 return (ENOTSUP);
2087         }
2088 
2089         /* Check to see if this is a library */
2090         if (hdr->a_magic == ZMAGIC && hdr->a_entry < PAGESIZE) {
2091                 is_library = 1;
2092         }
2093 
2094         /* Can't execute code from "noexec" mounted filesystem. */
2095         if (((vp->v_vfsp->vfs_flag & VFS_NOEXEC) != 0) && (is_library == 0)) {
2096                 MOBJ_STAT_ADD(aout_noexec);
2097                 return (EACCES);
2098         }
2099 
2100         /*
2101          * There are 2 ways to calculate the mapped size of executable:
2102          * 1) rounded text size + data size + bss size.
2103          * 2) starting offset for text + text size + data size + text relocation
2104          *    size + data relocation size + room for nlist data structure.
2105          *
2106          * The larger of the two sizes will be used to map this binary.
2107          */
2108         osize = P2ROUNDUP(hdr->a_text, PAGESIZE) + hdr->a_data + hdr->a_bss;
2109 
2110         off = hdr->a_magic == ZMAGIC ? 0 : sizeof (struct exec);
2111 
2112         nsize = off + hdr->a_text + hdr->a_data + hdr->a_trsize +
2113             hdr->a_drsize + NLIST_SIZE;
2114 
2115         size = MAX(osize, nsize);
2116         if (size != nsize) {
2117                 nsize = 0;
2118         }
2119 
2120         /*
2121          * 1 seg for text and 1 seg for initialized data.
2122          * 1 seg for bss (if can't fit in leftover space of init data)
2123          * 1 seg for nlist if needed.
2124          */
2125         to_map = 2 + (nsize ? 1 : 0) +
2126             (hdr->a_bss > PAGESIZE - P2PHASE(hdr->a_data, PAGESIZE) ? 1 : 0);
2127         if (*num_mapped < to_map) {
2128                 *num_mapped = to_map;
2129                 MOBJ_STAT_ADD(aout_e2big);
2130                 return (E2BIG);
2131         }
2132 
2133         /* Reserve address space for the whole mapping */
2134         if (is_library) {
2135                 /* We'll let VOP_MAP below pick our address for us */
2136                 addr = NULL;
2137                 MOBJ_STAT_ADD(aout_lib);
2138         } else {
2139                 /*
2140                  * default start address for fixed binaries from AOUT 4.x
2141                  * standard.
2142                  */
2143                 MOBJ_STAT_ADD(aout_fixed);
2144                 mflag |= MAP_FIXED;
2145                 addr = (caddr_t)0x2000;
2146                 as_rangelock(as);
2147                 if (as_gap(as, size, &addr, &size, 0, NULL) != 0) {
2148                         as_rangeunlock(as);
2149                         MOBJ_STAT_ADD(aout_addr_in_use);
2150                         return (EADDRINUSE);
2151                 }
2152                 crargs.flags |= MAP_NORESERVE;
2153                 error = as_map(as, addr, size, segvn_create, &crargs);
2154                 ASSERT(addr == (caddr_t)0x2000);
2155                 as_rangeunlock(as);
2156         }
2157 
2158         start_addr = addr;
2159         osize = size;
2160 
2161         /*
2162          * Map as large as we need, backed by file, this will be text, and
2163          * possibly the nlist segment.  We map over this mapping for bss and
2164          * initialized data segments.
2165          */
2166         error = VOP_MAP(vp, off, as, &addr, size, prot, PROT_ALL,
2167             mflag, fcred, NULL);
2168         if (error) {
2169                 if (!is_library) {
2170                         (void) as_unmap(as, start_addr, osize);
2171                 }
2172                 return (error);
2173         }
2174 
2175         /* pickup the value of start_addr and osize for libraries */
2176         start_addr = addr;
2177         osize = size;
2178 
2179         /*
2180          * We have our initial reservation/allocation so we need to use fixed
2181          * addresses from now on.
2182          */
2183         mflag |= MAP_FIXED;
2184 
2185         mrp[0].mr_addr = addr;
2186         mrp[0].mr_msize = hdr->a_text;
2187         mrp[0].mr_fsize = hdr->a_text;
2188         mrp[0].mr_offset = 0;
2189         mrp[0].mr_prot = PROT_READ | PROT_EXEC;
2190         mrp[0].mr_flags = MR_HDR_AOUT;
2191 
2192 
2193         /*
2194          * Map initialized data. We are mapping over a portion of the
2195          * previous mapping which will be unmapped in VOP_MAP below.
2196          */
2197         off = P2ROUNDUP((offset_t)(hdr->a_text), PAGESIZE);
2198         msize = off;
2199         addr += off;
2200         size = hdr->a_data;
2201         error = VOP_MAP(vp, off, as, &addr, size, PROT_ALL, PROT_ALL,
2202             mflag, fcred, NULL);
2203         if (error) {
2204                 (void) as_unmap(as, start_addr, osize);
2205                 return (error);
2206         }
2207         msize += size;
2208         mrp[1].mr_addr = addr;
2209         mrp[1].mr_msize = size;
2210         mrp[1].mr_fsize = size;
2211         mrp[1].mr_offset = 0;
2212         mrp[1].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
2213         mrp[1].mr_flags = 0;
2214 
2215         /* Need to zero out remainder of page */
2216         addr += hdr->a_data;
2217         zfoddiff = P2PHASE((size_t)addr, PAGESIZE);
2218         if (zfoddiff) {
2219                 label_t ljb;
2220 
2221                 MOBJ_STAT_ADD(aout_zfoddiff);
2222                 zfoddiff = PAGESIZE - zfoddiff;
2223                 if (on_fault(&ljb)) {
2224                         no_fault();
2225                         MOBJ_STAT_ADD(aout_uzero_fault);
2226                         (void) as_unmap(as, start_addr, osize);
2227                         return (EFAULT);
2228                 }
2229                 uzero(addr, zfoddiff);
2230                 no_fault();
2231         }
2232         msize += zfoddiff;
2233         segnum = 2;
2234 
2235         /* Map bss */
2236         if (hdr->a_bss > zfoddiff) {
2237                 struct segvn_crargs crargs =
2238                     SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);
2239                 MOBJ_STAT_ADD(aout_map_bss);
2240                 addr += zfoddiff;
2241                 size = hdr->a_bss - zfoddiff;
2242                 as_rangelock(as);
2243                 (void) as_unmap(as, addr, size);
2244                 error = as_map(as, addr, size, segvn_create, &crargs);
2245                 as_rangeunlock(as);
2246                 msize += size;
2247 
2248                 if (error) {
2249                         MOBJ_STAT_ADD(aout_bss_fail);
2250                         (void) as_unmap(as, start_addr, osize);
2251                         return (error);
2252                 }
2253                 mrp[2].mr_addr = addr;
2254                 mrp[2].mr_msize = size;
2255                 mrp[2].mr_fsize = 0;
2256                 mrp[2].mr_offset = 0;
2257                 mrp[2].mr_prot = PROT_READ | PROT_WRITE | PROT_EXEC;
2258                 mrp[2].mr_flags = 0;
2259 
2260                 addr += size;
2261                 segnum = 3;
2262         }
2263 
2264         /*
2265          * If we have extra bits left over, we need to include that in how
2266          * much we mapped to make sure the nlist logic is correct
2267          */
2268         msize = P2ROUNDUP(msize, PAGESIZE);
2269 
2270         if (nsize && msize < nsize) {
2271                 MOBJ_STAT_ADD(aout_nlist);
2272                 mrp[segnum].mr_addr = addr;
2273                 mrp[segnum].mr_msize = nsize - msize;
2274                 mrp[segnum].mr_fsize = 0;
2275                 mrp[segnum].mr_offset = 0;
2276                 mrp[segnum].mr_prot = PROT_READ | PROT_EXEC;
2277                 mrp[segnum].mr_flags = 0;
2278         }
2279 
2280         *num_mapped = to_map;
2281         return (0);
2282 }
2283 #endif
2284 
2285 /*
2286  * These are the two types of files that we can interpret and we want to read
2287  * in enough info to cover both types when looking at the initial header.
2288  */
2289 #define MAX_HEADER_SIZE (MAX(sizeof (Ehdr), sizeof (struct exec)))
2290 
2291 /*
2292  * Map vp passed in in an interpreted manner.  ELF and AOUT files will be
2293  * interpreted and mapped appropriately for execution.
2294  * num_mapped in - # elements in mrp
2295  * num_mapped out - # sections mapped and length of mrp array if
2296  *                  no errors or E2BIG returned.
2297  *
2298  * Returns 0 on success, errno value on failure.
2299  */
2300 static int
2301 mmapobj_map_interpret(vnode_t *vp, mmapobj_result_t *mrp,
2302     uint_t *num_mapped, size_t padding, cred_t *fcred)
2303 {
2304         int error = 0;
2305         vattr_t vattr;
2306         struct lib_va *lvp;
2307         caddr_t start_addr;
2308         model_t model;
2309 
2310         /*
2311          * header has to be aligned to the native size of ulong_t in order
2312          * to avoid an unaligned access when dereferencing the header as
2313          * a ulong_t.  Thus we allocate our array on the stack of type
2314          * ulong_t and then have header, which we dereference later as a char
2315          * array point at lheader.
2316          */
2317         ulong_t lheader[(MAX_HEADER_SIZE / (sizeof (ulong_t))) + 1];
2318         caddr_t header = (caddr_t)&lheader;
2319 
2320         vattr.va_mask = AT_FSID | AT_NODEID | AT_CTIME | AT_MTIME | AT_SIZE;
2321         error = VOP_GETATTR(vp, &vattr, 0, fcred, NULL);
2322         if (error) {
2323                 return (error);
2324         }
2325 
2326         /*
2327          * Check lib_va to see if we already have a full description
2328          * for this library.  This is the fast path and only used for
2329          * ET_DYN ELF files (dynamic libraries).
2330          */
2331         if (padding == 0 && !secflag_enabled(curproc, PROC_SEC_ASLR) &&
2332             ((lvp = lib_va_find(&vattr)) != NULL)) {
2333                 int num_segs;
2334 
2335                 model = get_udatamodel();
2336                 if ((model == DATAMODEL_ILP32 &&
2337                     lvp->lv_flags & LV_ELF64) ||
2338                     (model == DATAMODEL_LP64 &&
2339                     lvp->lv_flags & LV_ELF32)) {
2340                         lib_va_release(lvp);
2341                         MOBJ_STAT_ADD(fast_wrong_model);
2342                         return (ENOTSUP);
2343                 }
2344                 num_segs = lvp->lv_num_segs;
2345                 if (*num_mapped < num_segs) {
2346                         *num_mapped = num_segs;
2347                         lib_va_release(lvp);
2348                         MOBJ_STAT_ADD(fast_e2big);
2349                         return (E2BIG);
2350                 }
2351 
2352                 /*
2353                  * Check to see if we have all the mappable program headers
2354                  * cached.
2355                  */
2356                 if (num_segs <= LIBVA_CACHED_SEGS && num_segs != 0) {
2357                         MOBJ_STAT_ADD(fast);
2358                         start_addr = mmapobj_lookup_start_addr(lvp);
2359                         if (start_addr == NULL) {
2360                                 lib_va_release(lvp);
2361                                 return (ENOMEM);
2362                         }
2363 
2364                         bcopy(lvp->lv_mps, mrp,
2365                             num_segs * sizeof (mmapobj_result_t));
2366 
2367                         error = mmapobj_map_elf(vp, start_addr, mrp,
2368                             num_segs, fcred, ET_DYN);
2369 
2370                         lib_va_release(lvp);
2371                         if (error == 0) {
2372                                 *num_mapped = num_segs;
2373                                 MOBJ_STAT_ADD(fast_success);
2374                         }
2375                         return (error);
2376                 }
2377                 MOBJ_STAT_ADD(fast_not_now);
2378 
2379                 /* Release it for now since we'll look it up below */
2380                 lib_va_release(lvp);
2381         }
2382 
2383         /*
2384          * Time to see if this is a file we can interpret.  If it's smaller
2385          * than this, then we can't interpret it.
2386          */
2387         if (vattr.va_size < MAX_HEADER_SIZE) {
2388                 MOBJ_STAT_ADD(small_file);
2389                 return (ENOTSUP);
2390         }
2391 
2392         if ((error = vn_rdwr(UIO_READ, vp, header, MAX_HEADER_SIZE, 0,
2393             UIO_SYSSPACE, 0, (rlim64_t)0, fcred, NULL)) != 0) {
2394                 MOBJ_STAT_ADD(read_error);
2395                 return (error);
2396         }
2397 
2398         /* Verify file type */
2399         if (header[EI_MAG0] == ELFMAG0 && header[EI_MAG1] == ELFMAG1 &&
2400             header[EI_MAG2] == ELFMAG2 && header[EI_MAG3] == ELFMAG3) {
2401                 return (doelfwork((Ehdr *)lheader, vp, mrp, num_mapped,
2402                     padding, fcred));
2403         }
2404 
2405 #if defined(__sparc)
2406         /* On sparc, check for 4.X AOUT format */
2407         switch (((struct exec *)header)->a_magic) {
2408         case OMAGIC:
2409         case ZMAGIC:
2410         case NMAGIC:
2411                 return (doaoutwork(vp, mrp, num_mapped,
2412                     (struct exec *)lheader, fcred));
2413         }
2414 #endif
2415 
2416         /* Unsupported type */
2417         MOBJ_STAT_ADD(unsupported);
2418         return (ENOTSUP);
2419 }
2420 
2421 /*
2422  * Given a vnode, map it as either a flat file or interpret it and map
2423  * it according to the rules of the file type.
2424  * *num_mapped will contain the size of the mmapobj_result_t array passed in.
2425  * If padding is non-zero, the mappings will be padded by that amount
2426  * rounded up to the nearest pagesize.
2427  * If the mapping is successful, *num_mapped will contain the number of
2428  * distinct mappings created, and mrp will point to the array of
2429  * mmapobj_result_t's which describe these mappings.
2430  *
2431  * On error, -1 is returned and errno is set appropriately.
2432  * A special error case will set errno to E2BIG when there are more than
2433  * *num_mapped mappings to be created and *num_mapped will be set to the
2434  * number of mappings needed.
2435  */
2436 int
2437 mmapobj(vnode_t *vp, uint_t flags, mmapobj_result_t *mrp,
2438     uint_t *num_mapped, size_t padding, cred_t *fcred)
2439 {
2440         int to_map;
2441         int error = 0;
2442 
2443         ASSERT((padding & PAGEOFFSET) == 0);
2444         ASSERT((flags & ~MMOBJ_ALL_FLAGS) == 0);
2445         ASSERT(num_mapped != NULL);
2446         ASSERT((flags & MMOBJ_PADDING) ? padding != 0 : padding == 0);
2447 
2448         if ((flags & MMOBJ_INTERPRET) == 0) {
2449                 to_map = padding ? 3 : 1;
2450                 if (*num_mapped < to_map) {
2451                         *num_mapped = to_map;
2452                         MOBJ_STAT_ADD(flat_e2big);
2453                         return (E2BIG);
2454                 }
2455                 error = mmapobj_map_flat(vp, mrp, padding, fcred);
2456 
2457                 if (error) {
2458                         return (error);
2459                 }
2460                 *num_mapped = to_map;
2461                 return (0);
2462         }
2463 
2464         error = mmapobj_map_interpret(vp, mrp, num_mapped, padding, fcred);
2465         return (error);
2466 }