kernel/krtld/kobj.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
  27  * All rights reserved. Use is subject to license terms.
  28  * Copyright (c) 2018, Joyent, Inc.
  29  */
  30
  31 /*
  32  * Kernel's linker/loader
  33  */
  34
  35 #include <sys/types.h>
  36 #include <sys/param.h>
  37 #include <sys/sysmacros.h>
  38 #include <sys/systm.h>
  39 #include <sys/user.h>
  40 #include <sys/kmem.h>
  41 #include <sys/reboot.h>
  42 #include <sys/bootconf.h>
  43 #include <sys/debug.h>
  44 #include <sys/uio.h>
  45 #include <sys/file.h>
  46 #include <sys/vnode.h>
  47 #include <sys/user.h>
  48 #include <sys/mman.h>
  49 #include <vm/as.h>
  50 #include <vm/seg_kp.h>
  51 #include <vm/seg_kmem.h>
  52 #include <sys/elf.h>
  53 #include <sys/elf_notes.h>
  54 #include <sys/vmsystm.h>
  55 #include <sys/kdi.h>
  56 #include <sys/atomic.h>
  57 #include <sys/kmdb.h>
  58
  59 #include <sys/link.h>
  60 #include <sys/kobj.h>
  61 #include <sys/ksyms.h>
  62 #include <sys/disp.h>
  63 #include <sys/modctl.h>
  64 #include <sys/varargs.h>
  65 #include <sys/kstat.h>
  66 #include <sys/kobj_impl.h>
  67 #include <sys/fs/decomp.h>
  68 #include <sys/callb.h>
  69 #include <sys/cmn_err.h>
  70 #include <sys/tnf_probe.h>
  71 #include <sys/zmod.h>
  72
  73 #include <krtld/reloc.h>
  74 #include <krtld/kobj_kdi.h>
  75 #include <sys/sha1.h>
  76 #include <sys/crypto/elfsign.h>
  77
  78 #if !defined(_OBP)
  79 #include <sys/bootvfs.h>
  80 #endif
  81
  82 /*
  83  * do_symbols() error codes
  84  */
  85 #define DOSYM_UNDEF             -1      /* undefined symbol */
  86 #define DOSYM_UNSAFE            -2      /* MT-unsafe driver symbol */
  87
  88 #if !defined(_OBP)
  89 static void synthetic_bootaux(char *, val_t *);
  90 #endif
  91
  92 static struct module *load_exec(val_t *, char *);
  93 static void load_linker(val_t *);
  94 static struct modctl *add_primary(const char *filename, int);
  95 static int bind_primary(val_t *, int);
  96 static int load_primary(struct module *, int);
  97 static int load_kmdb(val_t *);
  98 static int get_progbits(struct module *, struct _buf *);
  99 static int get_syms(struct module *, struct _buf *);
 100 static int get_ctf(struct module *, struct _buf *);
 101 static void get_signature(struct module *, struct _buf *);
 102 static int do_common(struct module *);
 103 static void add_dependent(struct module *, struct module *);
 104 static int do_dependents(struct modctl *, char *, size_t);
 105 static int do_symbols(struct module *, Elf64_Addr);
 106 static void module_assign(struct modctl *, struct module *);
 107 static void free_module_data(struct module *);
 108 static char *depends_on(struct module *);
 109 static char *getmodpath(const char *);
 110 static char *basename(char *);
 111 static void attr_val(val_t *);
 112 static char *find_libmacro(char *);
 113 static char *expand_libmacro(char *, char *, char *);
 114 static int read_bootflags(void);
 115 static int kobj_comp_setup(struct _buf *, struct compinfo *);
 116 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
 117 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
 118 static int kobj_boot_open(char *, int);
 119 static int kobj_boot_close(int);
 120 static int kobj_boot_seek(int, off_t, off_t);
 121 static int kobj_boot_read(int, caddr_t, size_t);
 122 static int kobj_boot_fstat(int, struct bootstat *);
 123 static int kobj_boot_compinfo(int, struct compinfo *);
 124 extern int kobj_boot_mountroot(void);
 125
 126 static Sym *lookup_one(struct module *, const char *);
 127 static void sym_insert(struct module *, char *, symid_t);
 128 static Sym *sym_lookup(struct module *, Sym *);
 129
 130 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
 131 static void kobjopen_free(struct kobjopen_tctl *ltp);
 132 static void kobjopen_thread(struct kobjopen_tctl *ltp);
 133 static int kobj_is_compressed(intptr_t);
 134
 135 extern int kcopy(const void *, void *, size_t);
 136 extern int elf_mach_ok(Ehdr *);
 137 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
 138
 139 static void tnf_unsplice_probes(uint_t, struct modctl *);
 140 extern tnf_probe_control_t *__tnf_probe_list_head;
 141 extern tnf_tag_data_t *__tnf_tag_list_head;
 142
 143 extern int modrootloaded;
 144 extern int swaploaded;
 145 extern int bop_io_quiesced;
 146 extern int last_module_id;
 147
 148 extern char stubs_base[];
 149 extern char stubs_end[];
 150
 151 #ifdef KOBJ_DEBUG
 152 /*
 153  * Values that can be or'd in to kobj_debug and their effects:
 154  *
 155  *      D_DEBUG         - misc. debugging information.
 156  *      D_SYMBOLS       - list symbols and their values as they are entered
 157  *                        into the hash table
 158  *      D_RELOCATIONS   - display relocation processing information
 159  *      D_LOADING       - display information about each module as it
 160  *                        is loaded.
 161  */
 162 int kobj_debug = 0;
 163
 164 #define KOBJ_MARK(s)    if (kobj_debug & D_DEBUG)       \
 165         (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
 166 #else
 167 #define KOBJ_MARK(s)    /* discard */
 168 #endif
 169
 170 #define MODPATH_PROPNAME        "module-path"
 171
 172 #define _moddebug       get_weakish_int(&moddebug)
 173 #define _modrootloaded  get_weakish_int(&modrootloaded)
 174 #define _swaploaded     get_weakish_int(&swaploaded)
 175 #define _ioquiesced     get_weakish_int(&bop_io_quiesced)
 176
 177 #define mod(X)          (struct module *)((X)->modl_modp->mod_mp)
 178
 179 void    *romp;          /* rom vector (opaque to us) */
 180 struct bootops *ops;    /* bootops vector */
 181 void *dbvec;            /* debug vector */
 182
 183 /*
 184  * kobjopen thread control structure
 185  */
 186 struct kobjopen_tctl {
 187         ksema_t         sema;
 188         char            *name;          /* name of file */
 189         struct vnode    *vp;            /* vnode return from vn_open() */
 190         int             Errno;          /* error return from vnopen    */
 191 };
 192
 193 /*
 194  * Structure for defining dynamically expandable library macros
 195  */
 196
 197 struct lib_macro_info {
 198         char    *lmi_list;              /* ptr to list of possible choices */
 199         char    *lmi_macroname;         /* pointer to macro name */
 200         ushort_t lmi_ba_index;          /* index into bootaux vector */
 201         ushort_t lmi_macrolen;          /* macro length */
 202 } libmacros[] = {
 203         { NULL, "CPU", BA_CPU, 0 },
 204         { NULL, "MMU", BA_MMU, 0 }
 205 };
 206
 207 #define NLIBMACROS      sizeof (libmacros) / sizeof (struct lib_macro_info)
 208
 209 char *boot_cpu_compatible_list;                 /* make $CPU available */
 210
 211 char *kobj_module_path;                         /* module search path */
 212 vmem_t  *text_arena;                            /* module text arena */
 213 static vmem_t *data_arena;                      /* module data & bss arena */
 214 static vmem_t *ctf_arena;                       /* CTF debug data arena */
 215 static struct modctl *kobj_modules = NULL;      /* modules loaded */
 216 int kobj_mmu_pagesize;                          /* system pagesize */
 217 static int lg_pagesize;                         /* "large" pagesize */
 218 static int kobj_last_module_id = 0;             /* id assignment */
 219 static kmutex_t kobj_lock;                      /* protects mach memory list */
 220
 221 /*
 222  * The following functions have been implemented by the kernel.
 223  * However, many 3rd party drivers provide their own implementations
 224  * of these functions.  When such drivers are loaded, messages
 225  * indicating that these symbols have been multiply defined will be
 226  * emitted to the console.  To avoid alarming customers for no good
 227  * reason, we simply suppress such warnings for the following set of
 228  * functions.
 229  */
 230 static char *suppress_sym_list[] =
 231 {
 232         "strstr",
 233         "strncat",
 234         "strlcat",
 235         "strlcpy",
 236         "strspn",
 237         "memcpy",
 238         "memset",
 239         "memmove",
 240         "memcmp",
 241         "memchr",
 242         "__udivdi3",
 243         "__divdi3",
 244         "__umoddi3",
 245         "__moddi3",
 246         NULL            /* This entry must exist */
 247 };
 248
 249 /* indexed by KOBJ_NOTIFY_* */
 250 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
 251
 252 /*
 253  * TNF probe management globals
 254  */
 255 tnf_probe_control_t     *__tnf_probe_list_head = NULL;
 256 tnf_tag_data_t          *__tnf_tag_list_head = NULL;
 257 int                     tnf_changed_probe_list = 0;
 258
 259 /*
 260  * Prefix for statically defined tracing (SDT) DTrace probes.
 261  */
 262 const char              *sdt_prefix = "__dtrace_probe_";
 263
 264 /*
 265  * Beginning and end of the kernel's dynamic text/data segments.
 266  */
 267 static caddr_t _text;
 268 static caddr_t _etext;
 269 static caddr_t _data;
 270
 271 /*
 272  * The sparc linker doesn't create a memory location
 273  * for a variable named _edata, so _edata can only be
 274  * referred to, not modified.  krtld needs a static
 275  * variable to modify it - within krtld, of course -
 276  * outside of krtld, e_data is used in all kernels.
 277  */
 278 extern caddr_t _edata;
 279
 280 Addr dynseg = 0;        /* load address of "dynamic" segment */
 281 size_t dynsize;         /* "dynamic" segment size */
 282
 283
 284 int standalone = 1;                     /* an unwholey kernel? */
 285 int use_iflush;                         /* iflush after relocations */
 286
 287 /*
 288  * _kobj_printf() and _vkobj_printf()
 289  *
 290  * Common printf function pointer. Can handle only one conversion
 291  * specification in the format string. Some of the functions invoked
 292  * through this function pointer cannot handle more that one conversion
 293  * specification in the format string.
 294  */
 295 void (*_kobj_printf)(void *, const char *, ...);        /* printf routine */
 296 void (*_vkobj_printf)(void *, const char *, va_list);   /* vprintf routine */
 297
 298 /*
 299  * Standalone function pointers for use within krtld.
 300  * Many platforms implement optimized platmod versions of
 301  * utilities such as bcopy and any such are not yet available
 302  * until the kernel is more completely stitched together.
 303  * See kobj_impl.h
 304  */
 305 void (*kobj_bcopy)(const void *, void *, size_t);
 306 void (*kobj_bzero)(void *, size_t);
 307 size_t (*kobj_strlcat)(char *, const char *, size_t);
 308
 309 static kobj_stat_t kobj_stat;
 310
 311 #define MINALIGN        8       /* at least a double-word */
 312
 313 int
 314 get_weakish_int(int *ip)
 315 {
 316         if (standalone)
 317                 return (0);
 318         return (ip == NULL ? 0 : *ip);
 319 }
 320
 321 static void *
 322 get_weakish_pointer(void **ptrp)
 323 {
 324         if (standalone)
 325                 return (0);
 326         return (ptrp == NULL ? 0 : *ptrp);
 327 }
 328
 329 /*
 330  * XXX fix dependencies on "kernel"; this should work
 331  * for other standalone binaries as well.
 332  *
 333  * XXX Fix hashing code to use one pointer to
 334  * hash entries.
 335  *      |----------|
 336  *      | nbuckets |
 337  *      |----------|
 338  *      | nchains  |
 339  *      |----------|
 340  *      | bucket[] |
 341  *      |----------|
 342  *      | chain[]  |
 343  *      |----------|
 344  */
 345
 346 /*
 347  * Load, bind and relocate all modules that
 348  * form the primary kernel. At this point, our
 349  * externals have not been relocated.
 350  */
 351 void
 352 kobj_init(
 353         void *romvec,
 354         void *dvec,
 355         struct bootops *bootvec,
 356         val_t *bootaux)
 357 {
 358         struct module *mp;
 359         struct modctl *modp;
 360         Addr entry;
 361         char filename[MAXPATHLEN];
 362
 363         /*
 364          * Save these to pass on to
 365          * the booted standalone.
 366          */
 367         romp = romvec;
 368         dbvec = dvec;
 369
 370         ops = bootvec;
 371         kobj_setup_standalone_vectors();
 372
 373         KOBJ_MARK("Entered kobj_init()");
 374
 375         (void) BOP_GETPROP(ops, "whoami", filename);
 376
 377         /*
 378          * We don't support standalone debuggers anymore.  The use of kadb
 379          * will interfere with the later use of kmdb.  Let the user mend
 380          * their ways now.  Users will reach this message if they still
 381          * have the kadb binary on their system (perhaps they used an old
 382          * bfu, or maybe they intentionally copied it there) and have
 383          * specified its use in a way that eluded our checking in the boot
 384          * program.
 385          */
 386         if (dvec != NULL) {
 387                 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
 388                     "kadb are no longer supported\n\n");
 389                 goto fail;
 390         }
 391
 392 #if defined(_OBP)
 393         /*
 394          * OBP allows us to read both the ramdisk and
 395          * the underlying root fs when root is a disk.
 396          * This can lower incidences of unbootable systems
 397          * when the archive is out-of-date with the /etc
 398          * state files.
 399          */
 400         if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
 401                 _kobj_printf(ops, "can't mount boot fs\n");
 402                 goto fail;
 403         }
 404 #else
 405         {
 406                 /* on x86, we always boot with a ramdisk */
 407                 (void) kobj_boot_mountroot();
 408
 409                 /*
 410                  * Now that the ramdisk is mounted, finish boot property
 411                  * initialization.
 412                  */
 413                 boot_prop_finish();
 414         }
 415
 416         /*
 417          * The early boot code does -not- hand us any of the dynamic
 418          * metadata about the executable.  In particular, it does not read
 419          * in, map or otherwise look at the program headers.  We fake all
 420          * that up now.
 421          *
 422          * We do this early as DTrace static probes both call undefined
 423          * references.  We have to process those relocations before calling
 424          * any of them.
 425          *
 426          * OBP tells kobj_start() where the ELF image is in memory, so it
 427          * synthesized bootaux before kobj_init() was called
 428          */
 429         if (bootaux[BA_PHDR].ba_ptr == NULL)
 430                 synthetic_bootaux(filename, bootaux);
 431
 432 #endif  /* _OBP */
 433
 434         /*
 435          * Save the interesting attribute-values
 436          * (scanned by kobj_boot).
 437          */
 438         attr_val(bootaux);
 439
 440         /*
 441          * Set the module search path.
 442          */
 443         kobj_module_path = getmodpath(filename);
 444
 445         boot_cpu_compatible_list = find_libmacro("CPU");
 446
 447         /*
 448          * These two modules have actually been
 449          * loaded by boot, but we finish the job
 450          * by introducing them into the world of
 451          * loadable modules.
 452          */
 453
 454         mp = load_exec(bootaux, filename);
 455         load_linker(bootaux);
 456
 457         /*
 458          * Load all the primary dependent modules.
 459          */
 460         if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
 461                 goto fail;
 462
 463         /*
 464          * Glue it together.
 465          */
 466         if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
 467                 goto fail;
 468
 469         entry = bootaux[BA_ENTRY].ba_val;
 470
 471         /*
 472          * Get the boot flags
 473          */
 474         bootflags(ops);
 475
 476         if (boothowto & RB_VERBOSE)
 477                 kobj_lm_dump(KOBJ_LM_PRIMARY);
 478
 479         kobj_kdi_init();
 480
 481         if (boothowto & RB_KMDB) {
 482                 if (load_kmdb(bootaux) < 0)
 483                         goto fail;
 484         }
 485
 486         /*
 487          * Post setup.
 488          */
 489         s_text = _text;
 490         e_text = _etext;
 491         s_data = _data;
 492         e_data = _edata;
 493
 494         kobj_sync_instruction_memory(s_text, e_text - s_text);
 495
 496 #ifdef  KOBJ_DEBUG
 497         if (kobj_debug & D_DEBUG)
 498                 _kobj_printf(ops,
 499                     "krtld: transferring control to: 0x%p\n", entry);
 500 #endif
 501
 502         /*
 503          * Make sure the mod system knows about the modules already loaded.
 504          */
 505         last_module_id = kobj_last_module_id;
 506         bcopy(kobj_modules, &modules, sizeof (modules));
 507         modp = &modules;
 508         do {
 509                 if (modp->mod_next == kobj_modules)
 510                         modp->mod_next = &modules;
 511                 if (modp->mod_prev == kobj_modules)
 512                         modp->mod_prev = &modules;
 513         } while ((modp = modp->mod_next) != &modules);
 514
 515         standalone = 0;
 516
 517 #ifdef  KOBJ_DEBUG
 518         if (kobj_debug & D_DEBUG)
 519                 _kobj_printf(ops,
 520                     "krtld: really transferring control to: 0x%p\n", entry);
 521 #endif
 522
 523         /* restore printf/bcopy/bzero vectors before returning */
 524         kobj_restore_vectors();
 525
 526 #if defined(_DBOOT)
 527         /*
 528          * krtld was called from a dboot ELF section, the embedded
 529          * dboot code contains the real entry via bootaux
 530          */
 531         exitto((caddr_t)entry);
 532 #else
 533         /*
 534          * krtld was directly called from startup
 535          */
 536         return;
 537 #endif
 538
 539 fail:
 540
 541         _kobj_printf(ops, "krtld: error during initial load/link phase\n");
 542         _kobj_printf(ops, "\n");
 543         _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
 544             " for:\n");
 545         _kobj_printf(ops, "    %s\n", filename);
 546         _kobj_printf(ops, "in the boot archive. Please verify that this"
 547             " file\n");
 548         _kobj_printf(ops, "matches what is found in the boot archive.\n");
 549         bop_panic("Unable to boot");
 550 }
 551
 552 #if !defined(_OBP)
 553 /*
 554  * Synthesize additional metadata that describes the executable as
 555  * krtld's caller didn't do it.
 556  */
 557 static void
 558 synthetic_bootaux(char *filename, val_t *bootaux)
 559 {
 560         Ehdr ehdr;
 561         caddr_t phdrbase;
 562         struct _buf *file;
 563         int i, n;
 564
 565         /*
 566          * Elf header
 567          */
 568         KOBJ_MARK("synthetic_bootaux()");
 569         KOBJ_MARK(filename);
 570         file = kobj_open_file(filename);
 571         if (file == (struct _buf *)-1) {
 572                 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
 573                 return;
 574         }
 575         KOBJ_MARK("reading program headers");
 576         if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
 577                 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
 578                     filename);
 579                 return;
 580         }
 581
 582         /*
 583          * Program headers
 584          */
 585         bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
 586         bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
 587         n = ehdr.e_phentsize * ehdr.e_phnum;
 588
 589         phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
 590
 591         if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
 592                 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
 593                     filename);
 594                 return;
 595         }
 596         bootaux[BA_PHDR].ba_ptr = phdrbase;
 597         kobj_close_file(file);
 598         KOBJ_MARK("closed file");
 599
 600         /*
 601          * Find the dynamic section address
 602          */
 603         for (i = 0; i < ehdr.e_phnum; i++) {
 604                 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
 605
 606                 if (phdr->p_type == PT_DYNAMIC) {
 607                         bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
 608                         break;
 609                 }
 610         }
 611         KOBJ_MARK("synthetic_bootaux() done");
 612 }
 613 #endif  /* !_OBP */
 614
 615 /*
 616  * Set up any global information derived
 617  * from attribute/values in the boot or
 618  * aux vector.
 619  */
 620 static void
 621 attr_val(val_t *bootaux)
 622 {
 623         Phdr *phdr;
 624         int phnum, phsize;
 625         int i;
 626
 627         KOBJ_MARK("attr_val()");
 628         kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
 629         lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
 630         use_iflush = bootaux[BA_IFLUSH].ba_val;
 631
 632         phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
 633         phnum = bootaux[BA_PHNUM].ba_val;
 634         phsize = bootaux[BA_PHENT].ba_val;
 635         for (i = 0; i < phnum; i++) {
 636                 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
 637
 638                 if (phdr->p_type != PT_LOAD) {
 639                         continue;
 640                 }
 641                 /*
 642                  * Bounds of the various segments.
 643                  */
 644                 if (!(phdr->p_flags & PF_X)) {
 645 #if defined(_RELSEG)
 646                         /*
 647                          * sparc kernel puts the dynamic info
 648                          * into a separate segment, which is
 649                          * free'd in bop_fini()
 650                          */
 651                         ASSERT(phdr->p_vaddr != 0);
 652                         dynseg = phdr->p_vaddr;
 653                         dynsize = phdr->p_memsz;
 654 #else
 655                         ASSERT(phdr->p_vaddr == 0);
 656 #endif
 657                 } else {
 658                         if (phdr->p_flags & PF_W) {
 659                                 _data = (caddr_t)phdr->p_vaddr;
 660                                 _edata = _data + phdr->p_memsz;
 661                         } else {
 662                                 _text = (caddr_t)phdr->p_vaddr;
 663                                 _etext = _text + phdr->p_memsz;
 664                         }
 665                 }
 666         }
 667
 668         /* To do the kobj_alloc, _edata needs to be set. */
 669         for (i = 0; i < NLIBMACROS; i++) {
 670                 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
 671                         libmacros[i].lmi_list = kobj_alloc(
 672                             strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
 673                             1, KM_WAIT);
 674                         (void) strcpy(libmacros[i].lmi_list,
 675                             bootaux[libmacros[i].lmi_ba_index].ba_ptr);
 676                 }
 677                 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
 678         }
 679 }
 680
 681 /*
 682  * Set up the booted executable.
 683  */
 684 static struct module *
 685 load_exec(val_t *bootaux, char *filename)
 686 {
 687         struct modctl *cp;
 688         struct module *mp;
 689         Dyn *dyn;
 690         Sym *sp;
 691         int i, lsize, osize, nsize, allocsize;
 692         char *libname, *tmp;
 693         char path[MAXPATHLEN];
 694
 695 #ifdef KOBJ_DEBUG
 696         if (kobj_debug & D_DEBUG)
 697                 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
 698 #endif
 699
 700         KOBJ_MARK("add_primary");
 701         cp = add_primary(filename, KOBJ_LM_PRIMARY);
 702
 703         KOBJ_MARK("struct module");
 704         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
 705         cp->mod_mp = mp;
 706
 707         /*
 708          * We don't have the following information
 709          * since this module is an executable and not
 710          * a relocatable .o.
 711          */
 712         mp->symtbl_section = 0;
 713         mp->shdrs = NULL;
 714         mp->strhdr = NULL;
 715
 716         /*
 717          * Since this module is the only exception,
 718          * we cons up some section headers.
 719          */
 720         KOBJ_MARK("symhdr");
 721         mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
 722
 723         KOBJ_MARK("strhdr");
 724         mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
 725
 726         mp->symhdr->sh_type = SHT_SYMTAB;
 727         mp->strhdr->sh_type = SHT_STRTAB;
 728         /*
 729          * Scan the dynamic structure.
 730          */
 731         for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
 732             dyn->d_tag != DT_NULL; dyn++) {
 733                 switch (dyn->d_tag) {
 734                 case DT_SYMTAB:
 735                         mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
 736                         mp->symhdr->sh_addr = dyn->d_un.d_ptr;
 737                         break;
 738                 case DT_HASH:
 739                         mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
 740                         mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
 741                         break;
 742                 case DT_STRTAB:
 743                         mp->strings = (char *)dyn->d_un.d_ptr;
 744                         mp->strhdr->sh_addr = dyn->d_un.d_ptr;
 745                         break;
 746                 case DT_STRSZ:
 747                         mp->strhdr->sh_size = dyn->d_un.d_val;
 748                         break;
 749                 case DT_SYMENT:
 750                         mp->symhdr->sh_entsize = dyn->d_un.d_val;
 751                         break;
 752                 }
 753         }
 754
 755         /*
 756          * Collapse any DT_NEEDED entries into one string.
 757          */
 758         nsize = osize = 0;
 759         allocsize = MAXPATHLEN;
 760
 761         KOBJ_MARK("depends_on");
 762         mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
 763
 764         for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
 765             dyn->d_tag != DT_NULL; dyn++)
 766                 if (dyn->d_tag == DT_NEEDED) {
 767                         char *_lib;
 768
 769                         libname = mp->strings + dyn->d_un.d_val;
 770                         if (strchr(libname, '$') != NULL) {
 771                                 if ((_lib = expand_libmacro(libname,
 772                                     path, path)) != NULL)
 773                                         libname = _lib;
 774                                 else
 775                                         _kobj_printf(ops, "krtld: "
 776                                             "load_exec: fail to "
 777                                             "expand %s\n", libname);
 778                         }
 779                         lsize = strlen(libname);
 780                         nsize += lsize;
 781                         if (nsize + 1 > allocsize) {
 782                                 KOBJ_MARK("grow depends_on");
 783                                 tmp = kobj_alloc(allocsize + MAXPATHLEN,
 784                                     KM_WAIT);
 785                                 bcopy(mp->depends_on, tmp, osize);
 786                                 kobj_free(mp->depends_on, allocsize);
 787                                 mp->depends_on = tmp;
 788                                 allocsize += MAXPATHLEN;
 789                         }
 790                         bcopy(libname, mp->depends_on + osize, lsize);
 791                         *(mp->depends_on + nsize) = ' '; /* separate */
 792                         nsize++;
 793                         osize = nsize;
 794                 }
 795         if (nsize) {
 796                 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
 797                 /*
 798                  * alloc with exact size and copy whatever it got over
 799                  */
 800                 KOBJ_MARK("realloc depends_on");
 801                 tmp = kobj_alloc(nsize, KM_WAIT);
 802                 bcopy(mp->depends_on, tmp, nsize);
 803                 kobj_free(mp->depends_on, allocsize);
 804                 mp->depends_on = tmp;
 805         } else {
 806                 kobj_free(mp->depends_on, allocsize);
 807                 mp->depends_on = NULL;
 808         }
 809
 810         mp->flags = KOBJ_EXEC|KOBJ_PRIM;        /* NOT a relocatable .o */
 811         mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
 812         /*
 813          * We allocate our own table since we don't
 814          * hash undefined references.
 815          */
 816         KOBJ_MARK("chains");
 817         mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
 818         KOBJ_MARK("buckets");
 819         mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
 820
 821         mp->text = _text;
 822         mp->data = _data;
 823
 824         mp->text_size = _etext - _text;
 825         mp->data_size = _edata - _data;
 826
 827         cp->mod_text = mp->text;
 828         cp->mod_text_size = mp->text_size;
 829
 830         mp->filename = cp->mod_filename;
 831
 832 #ifdef  KOBJ_DEBUG
 833         if (kobj_debug & D_LOADING) {
 834                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
 835                 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
 836                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
 837                 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
 838                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
 839         }
 840 #endif /* KOBJ_DEBUG */
 841
 842         /*
 843          * Insert symbols into the hash table.
 844          */
 845         for (i = 0; i < mp->nsyms; i++) {
 846                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
 847
 848                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
 849                         continue;
 850
 851                 sym_insert(mp, mp->strings + sp->st_name, i);
 852         }
 853
 854         KOBJ_MARK("load_exec done");
 855         return (mp);
 856 }
 857
 858 /*
 859  * Set up the linker module (if it's compiled in, LDNAME is NULL)
 860  */
 861 static void
 862 load_linker(val_t *bootaux)
 863 {
 864         struct module *kmp = (struct module *)kobj_modules->mod_mp;
 865         struct module *mp;
 866         struct modctl *cp;
 867         int i;
 868         Shdr *shp;
 869         Sym *sp;
 870         int shsize;
 871         char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
 872
 873         /*
 874          * On some architectures, krtld is compiled into the kernel.
 875          */
 876         if (dlname == NULL)
 877                 return;
 878
 879         cp = add_primary(dlname, KOBJ_LM_PRIMARY);
 880
 881         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
 882
 883         cp->mod_mp = mp;
 884         mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
 885         shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
 886         mp->shdrs = kobj_alloc(shsize, KM_WAIT);
 887         bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
 888
 889         for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
 890                 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
 891
 892                 if (shp->sh_flags & SHF_ALLOC) {
 893                         if (shp->sh_flags & SHF_WRITE) {
 894                                 if (mp->data == NULL)
 895                                         mp->data = (char *)shp->sh_addr;
 896                         } else if (mp->text == NULL) {
 897                                 mp->text = (char *)shp->sh_addr;
 898                         }
 899                 }
 900                 if (shp->sh_type == SHT_SYMTAB) {
 901                         mp->symtbl_section = i;
 902                         mp->symhdr = shp;
 903                         mp->symspace = mp->symtbl = (char *)shp->sh_addr;
 904                 }
 905         }
 906         mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
 907         mp->flags = KOBJ_INTERP|KOBJ_PRIM;
 908         mp->strhdr = (Shdr *)
 909             (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
 910         mp->strings = (char *)mp->strhdr->sh_addr;
 911         mp->hashsize = kobj_gethashsize(mp->nsyms);
 912
 913         mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
 914             (mp->hashsize + mp->nsyms) * sizeof (symid_t);
 915
 916         mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
 917         mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
 918
 919         mp->bss = bootaux[BA_BSS].ba_val;
 920         mp->bss_align = 0;      /* pre-aligned during allocation */
 921         mp->bss_size = (uintptr_t)_edata - mp->bss;
 922         mp->text_size = _etext - mp->text;
 923         mp->data_size = _edata - mp->data;
 924         mp->filename = cp->mod_filename;
 925         cp->mod_text = mp->text;
 926         cp->mod_text_size = mp->text_size;
 927
 928         /*
 929          * Now that we've figured out where the linker is,
 930          * set the limits for the booted object.
 931          */
 932         kmp->text_size = (size_t)(mp->text - kmp->text);
 933         kmp->data_size = (size_t)(mp->data - kmp->data);
 934         kobj_modules->mod_text_size = kmp->text_size;
 935
 936 #ifdef  KOBJ_DEBUG
 937         if (kobj_debug & D_LOADING) {
 938                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
 939                 _kobj_printf(ops, "\ttext:0x%p", mp->text);
 940                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
 941                 _kobj_printf(ops, "\tdata:0x%p", mp->data);
 942                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
 943         }
 944 #endif /* KOBJ_DEBUG */
 945
 946         /*
 947          * Insert the symbols into the hash table.
 948          */
 949         for (i = 0; i < mp->nsyms; i++) {
 950                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
 951
 952                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
 953                         continue;
 954                 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
 955                         if (sp->st_shndx == SHN_COMMON)
 956                                 sp->st_shndx = SHN_ABS;
 957                 }
 958                 sym_insert(mp, mp->strings + sp->st_name, i);
 959         }
 960
 961 }
 962
 963 static kobj_notify_list_t **
 964 kobj_notify_lookup(uint_t type)
 965 {
 966         ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
 967             sizeof (kobj_notify_list_t *));
 968
 969         return (&kobj_notifiers[type]);
 970 }
 971
 972 int
 973 kobj_notify_add(kobj_notify_list_t *knp)
 974 {
 975         kobj_notify_list_t **knl;
 976
 977         knl = kobj_notify_lookup(knp->kn_type);
 978
 979         knp->kn_next = NULL;
 980         knp->kn_prev = NULL;
 981
 982         mutex_enter(&kobj_lock);
 983
 984         if (*knl != NULL) {
 985                 (*knl)->kn_prev = knp;
 986                 knp->kn_next = *knl;
 987         }
 988         (*knl) = knp;
 989
 990         mutex_exit(&kobj_lock);
 991         return (0);
 992 }
 993
 994 int
 995 kobj_notify_remove(kobj_notify_list_t *knp)
 996 {
 997         kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
 998         kobj_notify_list_t *tknp;
 999
1000         mutex_enter(&kobj_lock);
1001
1002         /* LINTED */
1003         if (tknp = knp->kn_next)
1004                 tknp->kn_prev = knp->kn_prev;
1005
1006         /* LINTED */
1007         if (tknp = knp->kn_prev)
1008                 tknp->kn_next = knp->kn_next;
1009         else
1010                 *knl = knp->kn_next;
1011
1012         mutex_exit(&kobj_lock);
1013
1014         return (0);
1015 }
1016
1017 /*
1018  * Notify all interested callbacks of a specified change in module state.
1019  */
1020 static void
1021 kobj_notify(int type, struct modctl *modp)
1022 {
1023         kobj_notify_list_t *knp;
1024
1025         if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1026                 return;
1027
1028         mutex_enter(&kobj_lock);
1029
1030         for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1031                 knp->kn_func(type, modp);
1032
1033         /*
1034          * KDI notification must be last (it has to allow for work done by the
1035          * other notification callbacks), so we call it manually.
1036          */
1037         kobj_kdi_mod_notify(type, modp);
1038
1039         mutex_exit(&kobj_lock);
1040 }
1041
1042 /*
1043  * Create the module path.
1044  */
1045 static char *
1046 getmodpath(const char *filename)
1047 {
1048         char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1049
1050         /*
1051          * Platform code gets first crack, then add
1052          * the default components
1053          */
1054         mach_modpath(path, filename);
1055         if (*path != '\0')
1056                 (void) strcat(path, " ");
1057         return (strcat(path, MOD_DEFPATH));
1058 }
1059
1060 static struct modctl *
1061 add_primary(const char *filename, int lmid)
1062 {
1063         struct modctl *cp;
1064
1065         cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1066
1067         cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1068
1069         /*
1070          * For symbol lookup, we assemble our own
1071          * modctl list of the primary modules.
1072          */
1073
1074         (void) strcpy(cp->mod_filename, filename);
1075         cp->mod_modname = basename(cp->mod_filename);
1076
1077         /* set values for modinfo assuming that the load will work */
1078         cp->mod_prim = 1;
1079         cp->mod_loaded = 1;
1080         cp->mod_installed = 1;
1081         cp->mod_loadcnt = 1;
1082         cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1083
1084         cp->mod_id = kobj_last_module_id++;
1085
1086         /*
1087          * Link the module in. We'll pass this info on
1088          * to the mod squad later.
1089          */
1090         if (kobj_modules == NULL) {
1091                 kobj_modules = cp;
1092                 cp->mod_prev = cp->mod_next = cp;
1093         } else {
1094                 cp->mod_prev = kobj_modules->mod_prev;
1095                 cp->mod_next = kobj_modules;
1096                 kobj_modules->mod_prev->mod_next = cp;
1097                 kobj_modules->mod_prev = cp;
1098         }
1099
1100         kobj_lm_append(lmid, cp);
1101
1102         return (cp);
1103 }
1104
1105 static int
1106 bind_primary(val_t *bootaux, int lmid)
1107 {
1108         struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1109         struct modctl_list *lp;
1110         struct module *mp;
1111
1112         /*
1113          * Do common symbols.
1114          */
1115         for (lp = linkmap; lp; lp = lp->modl_next) {
1116                 mp = mod(lp);
1117
1118                 /*
1119                  * Don't do common section relocations for modules that
1120                  * don't need it.
1121                  */
1122                 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1123                         continue;
1124
1125                 if (do_common(mp) < 0)
1126                         return (-1);
1127         }
1128
1129         /*
1130          * Resolve symbols.
1131          */
1132         for (lp = linkmap; lp; lp = lp->modl_next) {
1133                 mp = mod(lp);
1134
1135                 if (do_symbols(mp, 0) < 0)
1136                         return (-1);
1137         }
1138
1139         /*
1140          * Do relocations.
1141          */
1142         for (lp = linkmap; lp; lp = lp->modl_next) {
1143                 mp = mod(lp);
1144
1145                 if (mp->flags & KOBJ_EXEC) {
1146                         Dyn *dyn;
1147                         Word relasz = 0, relaent = 0;
1148                         Word shtype;
1149                         char *rela = NULL;
1150
1151                         for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1152                             dyn->d_tag != DT_NULL; dyn++) {
1153                                 switch (dyn->d_tag) {
1154                                 case DT_RELASZ:
1155                                 case DT_RELSZ:
1156                                         relasz = dyn->d_un.d_val;
1157                                         break;
1158                                 case DT_RELAENT:
1159                                 case DT_RELENT:
1160                                         relaent = dyn->d_un.d_val;
1161                                         break;
1162                                 case DT_RELA:
1163                                         shtype = SHT_RELA;
1164                                         rela = (char *)dyn->d_un.d_ptr;
1165                                         break;
1166                                 case DT_REL:
1167                                         shtype = SHT_REL;
1168                                         rela = (char *)dyn->d_un.d_ptr;
1169                                         break;
1170                                 }
1171                         }
1172                         if (relasz == 0 ||
1173                             relaent == 0 || rela == NULL) {
1174                                 _kobj_printf(ops, "krtld: bind_primary(): "
1175                                     "no relocation information found for "
1176                                     "module %s\n", mp->filename);
1177                                 return (-1);
1178                         }
1179 #ifdef  KOBJ_DEBUG
1180                         if (kobj_debug & D_RELOCATIONS)
1181                                 _kobj_printf(ops, "krtld: relocating: file=%s "
1182                                     "KOBJ_EXEC\n", mp->filename);
1183 #endif
1184                         if (do_relocate(mp, rela, shtype, relasz/relaent,
1185                             relaent, (Addr)mp->text) < 0)
1186                                 return (-1);
1187                 } else {
1188                         if (do_relocations(mp) < 0)
1189                                 return (-1);
1190                 }
1191
1192                 kobj_sync_instruction_memory(mp->text, mp->text_size);
1193         }
1194
1195         for (lp = linkmap; lp; lp = lp->modl_next) {
1196                 mp = mod(lp);
1197
1198                 /*
1199                  * We need to re-read the full symbol table for the boot file,
1200                  * since we couldn't use the full one before.  We also need to
1201                  * load the CTF sections of both the boot file and the
1202                  * interpreter (us).
1203                  */
1204                 if (mp->flags & KOBJ_EXEC) {
1205                         struct _buf *file;
1206                         int n;
1207
1208                         file = kobj_open_file(mp->filename);
1209                         if (file == (struct _buf *)-1)
1210                                 return (-1);
1211                         if (kobj_read_file(file, (char *)&mp->hdr,
1212                             sizeof (mp->hdr), 0) < 0)
1213                                 return (-1);
1214                         n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1215                         mp->shdrs = kobj_alloc(n, KM_WAIT);
1216                         if (kobj_read_file(file, mp->shdrs, n,
1217                             mp->hdr.e_shoff) < 0)
1218                                 return (-1);
1219                         if (get_syms(mp, file) < 0)
1220                                 return (-1);
1221                         if (get_ctf(mp, file) < 0)
1222                                 return (-1);
1223                         kobj_close_file(file);
1224                         mp->flags |= KOBJ_RELOCATED;
1225
1226                 } else if (mp->flags & KOBJ_INTERP) {
1227                         struct _buf *file;
1228
1229                         file = kobj_open_path(mp->filename, 1);
1230                         if (file == (struct _buf *)-1)
1231                                 return (-1);
1232                         if (get_ctf(mp, file) < 0)
1233                                 return (-1);
1234                         kobj_close_file(file);
1235                         mp->flags |= KOBJ_RELOCATED;
1236                 }
1237         }
1238
1239         return (0);
1240 }
1241
1242 static struct modctl *
1243 mod_already_loaded(char *modname)
1244 {
1245         struct modctl *mctl = kobj_modules;
1246
1247         do {
1248                 if (strcmp(modname, mctl->mod_filename) == 0)
1249                         return (mctl);
1250                 mctl = mctl->mod_next;
1251
1252         } while (mctl != kobj_modules);
1253
1254         return (NULL);
1255 }
1256
1257 /*
1258  * Load all the primary dependent modules.
1259  */
1260 static int
1261 load_primary(struct module *mp, int lmid)
1262 {
1263         struct modctl *cp;
1264         struct module *dmp;
1265         char *p, *q;
1266         char modname[MODMAXNAMELEN];
1267
1268         if ((p = mp->depends_on) == NULL)
1269                 return (0);
1270
1271         /* CONSTANTCONDITION */
1272         while (1) {
1273                 /*
1274                  * Skip space.
1275                  */
1276                 while (*p && (*p == ' ' || *p == '\t'))
1277                         p++;
1278                 /*
1279                  * Get module name.
1280                  */
1281                 q = modname;
1282                 while (*p && *p != ' ' && *p != '\t')
1283                         *q++ = *p++;
1284
1285                 if (q == modname)
1286                         break;
1287
1288                 *q = '\0';
1289                 /*
1290                  * Check for dup dependencies.
1291                  */
1292                 if (strcmp(modname, "dtracestubs") == 0 ||
1293                     mod_already_loaded(modname) != NULL)
1294                         continue;
1295
1296                 cp = add_primary(modname, lmid);
1297                 cp->mod_busy = 1;
1298                 /*
1299                  * Load it.
1300                  */
1301                 (void) kobj_load_module(cp, 1);
1302                 cp->mod_busy = 0;
1303
1304                 if ((dmp = cp->mod_mp) == NULL) {
1305                         cp->mod_loaded = 0;
1306                         cp->mod_installed = 0;
1307                         cp->mod_loadcnt = 0;
1308                         return (-1);
1309                 }
1310
1311                 add_dependent(mp, dmp);
1312                 dmp->flags |= KOBJ_PRIM;
1313
1314                 /*
1315                  * Recurse.
1316                  */
1317                 if (load_primary(dmp, lmid) == -1) {
1318                         cp->mod_loaded = 0;
1319                         cp->mod_installed = 0;
1320                         cp->mod_loadcnt = 0;
1321                         return (-1);
1322                 }
1323         }
1324         return (0);
1325 }
1326
1327 static int
1328 console_is_usb_serial(void)
1329 {
1330         char *console;
1331         int len, ret;
1332
1333         if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1334                 return (0);
1335
1336         console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1337         (void) BOP_GETPROP(ops, "console", console);
1338         ret = (strcmp(console, "usb-serial") == 0);
1339         kobj_free(console, len);
1340
1341         return (ret);
1342 }
1343
1344 static int
1345 load_kmdb(val_t *bootaux)
1346 {
1347         struct modctl *mctl;
1348         struct module *mp;
1349         Sym *sym;
1350
1351         if (console_is_usb_serial()) {
1352                 _kobj_printf(ops, "kmdb not loaded "
1353                     "(unsupported on usb serial console)\n");
1354                 return (0);
1355         }
1356
1357         _kobj_printf(ops, "Loading kmdb...\n");
1358
1359         if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1360                 return (-1);
1361
1362         mctl->mod_busy = 1;
1363         (void) kobj_load_module(mctl, 1);
1364         mctl->mod_busy = 0;
1365
1366         if ((mp = mctl->mod_mp) == NULL)
1367                 return (-1);
1368
1369         mp->flags |= KOBJ_PRIM;
1370
1371         if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1372                 return (-1);
1373
1374         if (boothowto & RB_VERBOSE)
1375                 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1376
1377         if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1378                 return (-1);
1379
1380         if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1381                 return (-1);
1382
1383 #ifdef  KOBJ_DEBUG
1384         if (kobj_debug & D_DEBUG) {
1385                 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1386                     sym->st_value);
1387                 _kobj_printf(ops, "\tops 0x%p\n", ops);
1388                 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1389         }
1390 #endif
1391
1392         if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1393             (const char **)kobj_kmdb_argv) < 0)
1394                 return (-1);
1395
1396         return (0);
1397 }
1398
1399 /*
1400  * Return a string listing module dependencies.
1401  */
1402 static char *
1403 depends_on(struct module *mp)
1404 {
1405         Sym *sp;
1406         char *depstr, *q;
1407
1408         /*
1409          * The module doesn't have a depends_on value, so let's try it the
1410          * old-fashioned way - via "_depends_on"
1411          */
1412         if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1413                 return (NULL);
1414
1415         q = (char *)sp->st_value;
1416
1417 #ifdef KOBJ_DEBUG
1418         /*
1419          * _depends_on is a deprecated interface, so we warn about its use
1420          * irrespective of subsequent processing errors. How else are we going
1421          * to be able to deco this interface completely?
1422          * Changes initially limited to DEBUG because third-party modules
1423          * should be flagged to developers before general use base.
1424          */
1425         _kobj_printf(ops,
1426             "Warning: %s uses deprecated _depends_on interface.\n",
1427             mp->filename);
1428         _kobj_printf(ops, "Please notify module developer or vendor.\n");
1429 #endif
1430
1431         /*
1432          * Idiot checks. Make sure it's
1433          * in-bounds and NULL terminated.
1434          */
1435         if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1436                 _kobj_printf(ops, "Error processing dependency for %s\n",
1437                     mp->filename);
1438                 return (NULL);
1439         }
1440
1441         depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1442         (void) strcpy(depstr, q);
1443
1444         return (depstr);
1445 }
1446
1447 void
1448 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1449 {
1450         struct module *mp;
1451         mp = (struct module *)xmp;
1452
1453         modinfo->mi_base = mp->text;
1454         modinfo->mi_size = mp->text_size + mp->data_size;
1455 }
1456
1457 /*
1458  * kobj_export_ksyms() performs the following services:
1459  *
1460  * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1461  * (2) Removes unneeded symbols to save space.
1462  * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1463  * (4) Makes the symbol table visible to /dev/ksyms.
1464  */
1465 static void
1466 kobj_export_ksyms(struct module *mp)
1467 {
1468         Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1469         Sym *sp, *osp;
1470         char *name;
1471         size_t namelen;
1472         struct module *omp;
1473         uint_t nsyms;
1474         size_t symsize = mp->symhdr->sh_entsize;
1475         size_t locals = 1;
1476         size_t strsize;
1477
1478         /*
1479          * Make a copy of the original module structure.
1480          */
1481         omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1482         bcopy(mp, omp, sizeof (struct module));
1483
1484         /*
1485          * Compute the sizes of the new symbol table sections.
1486          */
1487         for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1488                 if (osp->st_value == 0)
1489                         continue;
1490                 if (sym_lookup(omp, osp) == NULL)
1491                         continue;
1492                 name = omp->strings + osp->st_name;
1493                 namelen = strlen(name);
1494                 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1495                         locals++;
1496                 nsyms++;
1497                 strsize += namelen + 1;
1498         }
1499
1500         mp->nsyms = nsyms;
1501         mp->hashsize = kobj_gethashsize(mp->nsyms);
1502
1503         /*
1504          * ksyms_lock must be held as writer during any operation that
1505          * modifies ksyms_arena, including allocation from same, and
1506          * must not be dropped until the arena is vmem_walk()able.
1507          */
1508         rw_enter(&ksyms_lock, RW_WRITER);
1509
1510         /*
1511          * Allocate space for the new section headers (symtab and strtab),
1512          * symbol table, buckets, chains, and strings.
1513          */
1514         mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1515             (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1516
1517         if (mp->flags & KOBJ_NOKSYMS) {
1518                 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1519         } else {
1520                 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1521                     VM_BESTFIT | VM_SLEEP);
1522         }
1523         bzero(mp->symspace, mp->symsize);
1524
1525         /*
1526          * Divvy up symspace.
1527          */
1528         mp->shdrs = mp->symspace;
1529         mp->symhdr = (Shdr *)mp->shdrs;
1530         mp->strhdr = (Shdr *)(mp->symhdr + 1);
1531         mp->symtbl = (char *)(mp->strhdr + 1);
1532         mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1533         mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1534         mp->strings = (char *)(mp->chains + nsyms);
1535
1536         /*
1537          * Fill in the new section headers (symtab and strtab).
1538          */
1539         mp->hdr.e_shnum = 2;
1540         mp->symtbl_section = 0;
1541
1542         mp->symhdr->sh_type = SHT_SYMTAB;
1543         mp->symhdr->sh_addr = (Addr)mp->symtbl;
1544         mp->symhdr->sh_size = nsyms * symsize;
1545         mp->symhdr->sh_link = 1;
1546         mp->symhdr->sh_info = locals;
1547         mp->symhdr->sh_addralign = sizeof (Addr);
1548         mp->symhdr->sh_entsize = symsize;
1549
1550         mp->strhdr->sh_type = SHT_STRTAB;
1551         mp->strhdr->sh_addr = (Addr)mp->strings;
1552         mp->strhdr->sh_size = strsize;
1553         mp->strhdr->sh_addralign = 1;
1554
1555         /*
1556          * Construct the new symbol table.
1557          */
1558         for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1559                 if (osp->st_value == 0)
1560                         continue;
1561                 if (sym_lookup(omp, osp) == NULL)
1562                         continue;
1563                 name = omp->strings + osp->st_name;
1564                 namelen = strlen(name);
1565                 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1566                 bcopy(osp, sp, symsize);
1567                 bcopy(name, mp->strings + strsize, namelen);
1568                 sp->st_name = strsize;
1569                 sym_insert(mp, name, nsyms);
1570                 nsyms++;
1571                 strsize += namelen + 1;
1572         }
1573
1574         rw_exit(&ksyms_lock);
1575
1576         /*
1577          * Free the old section headers -- we'll never need them again.
1578          */
1579         if (!(mp->flags & KOBJ_PRIM)) {
1580                 uint_t  shn;
1581                 Shdr    *shp;
1582
1583                 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1584                         shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1585                         switch (shp->sh_type) {
1586                         case SHT_RELA:
1587                         case SHT_REL:
1588                                 if (shp->sh_addr != (uintptr_t)NULL) {
1589                                         kobj_free((void *)shp->sh_addr,
1590                                             shp->sh_size);
1591                                 }
1592                                 break;
1593                         }
1594                 }
1595                 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1596         }
1597         /*
1598          * Discard the old symbol table and our copy of the module strucure.
1599          */
1600         if (!(mp->flags & KOBJ_PRIM))
1601                 kobj_free(omp->symspace, omp->symsize);
1602         kobj_free(omp, sizeof (struct module));
1603 }
1604
1605 static void
1606 kobj_export_ctf(struct module *mp)
1607 {
1608         char *data = mp->ctfdata;
1609         size_t size = mp->ctfsize;
1610
1611         if (data != NULL) {
1612                 if (_moddebug & MODDEBUG_NOCTF) {
1613                         mp->ctfdata = NULL;
1614                         mp->ctfsize = 0;
1615                 } else {
1616                         mp->ctfdata = vmem_alloc(ctf_arena, size,
1617                             VM_BESTFIT | VM_SLEEP);
1618                         bcopy(data, mp->ctfdata, size);
1619                 }
1620
1621                 if (!(mp->flags & KOBJ_PRIM))
1622                         kobj_free(data, size);
1623         }
1624 }
1625
1626 void
1627 kobj_export_module(struct module *mp)
1628 {
1629         kobj_export_ksyms(mp);
1630         kobj_export_ctf(mp);
1631
1632         mp->flags |= KOBJ_EXPORTED;
1633 }
1634
1635 static int
1636 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1637 {
1638         char *path = NULL, *depstr = NULL;
1639         int allocsize = 0, osize = 0, nsize = 0;
1640         char *libname, *tmp;
1641         int lsize;
1642         Dyn *dynp;
1643
1644         for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1645                 switch (dynp->d_tag) {
1646                 case DT_NEEDED:
1647                         /*
1648                          * Read the DT_NEEDED entries, expanding the macros they
1649                          * contain (if any), and concatenating them into a
1650                          * single space-separated dependency list.
1651                          */
1652                         libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1653
1654                         if (strchr(libname, '$') != NULL) {
1655                                 char *_lib;
1656
1657                                 if (path == NULL)
1658                                         path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1659                                 if ((_lib = expand_libmacro(libname, path,
1660                                     path)) != NULL)
1661                                         libname = _lib;
1662                                 else {
1663                                         _kobj_printf(ops, "krtld: "
1664                                             "process_dynamic: failed to expand "
1665                                             "%s\n", libname);
1666                                 }
1667                         }
1668
1669                         lsize = strlen(libname);
1670                         nsize += lsize;
1671                         if (nsize + 1 > allocsize) {
1672                                 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1673                                     KM_WAIT);
1674                                 if (depstr != NULL) {
1675                                         bcopy(depstr, tmp, osize);
1676                                         kobj_free(depstr, allocsize);
1677                                 }
1678                                 depstr = tmp;
1679                                 allocsize += MAXPATHLEN;
1680                         }
1681                         bcopy(libname, depstr + osize, lsize);
1682                         *(depstr + nsize) = ' '; /* separator */
1683                         nsize++;
1684                         osize = nsize;
1685                         break;
1686
1687                 case DT_FLAGS_1:
1688                         if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1689                                 mp->flags |= KOBJ_IGNMULDEF;
1690                         if (dynp->d_un.d_val & DF_1_NOKSYMS)
1691                                 mp->flags |= KOBJ_NOKSYMS;
1692
1693                         break;
1694                 }
1695         }
1696
1697         /*
1698          * finish up the depends string (if any)
1699          */
1700         if (depstr != NULL) {
1701                 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1702                 if (path != NULL)
1703                         kobj_free(path, MAXPATHLEN);
1704
1705                 tmp = kobj_alloc(nsize, KM_WAIT);
1706                 bcopy(depstr, tmp, nsize);
1707                 kobj_free(depstr, allocsize);
1708                 depstr = tmp;
1709
1710                 mp->depends_on = depstr;
1711         }
1712
1713         return (0);
1714 }
1715
1716 static int
1717 do_dynamic(struct module *mp, struct _buf *file)
1718 {
1719         Shdr *dshp, *dstrp, *shp;
1720         char *dyndata, *dstrdata;
1721         int dshn, shn, rc;
1722
1723         /* find and validate the dynamic section (if any) */
1724
1725         for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1726                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1727                 switch (shp->sh_type) {
1728                 case SHT_DYNAMIC:
1729                         if (dshp != NULL) {
1730                                 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1731                                     mp->filename);
1732                                 _kobj_printf(ops,
1733                                     "multiple dynamic sections\n");
1734                                 return (-1);
1735                         } else {
1736                                 dshp = shp;
1737                                 dshn = shn;
1738                         }
1739                         break;
1740                 }
1741         }
1742
1743         if (dshp == NULL)
1744                 return (0);
1745
1746         if (dshp->sh_link > mp->hdr.e_shnum) {
1747                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1748                 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1749                 return (-1);
1750         }
1751         dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1752
1753         if (dstrp->sh_type != SHT_STRTAB) {
1754                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1755                 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1756                     dshn);
1757                 return (-1);
1758         }
1759
1760         /* read it from disk */
1761
1762         dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1763         if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1764                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1765                 _kobj_printf(ops, "error reading section %d\n", dshn);
1766
1767                 kobj_free(dyndata, dshp->sh_size);
1768                 return (-1);
1769         }
1770
1771         dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1772         if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1773             dstrp->sh_offset) < 0) {
1774                 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1775                 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1776
1777                 kobj_free(dyndata, dshp->sh_size);
1778                 kobj_free(dstrdata, dstrp->sh_size);
1779                 return (-1);
1780         }
1781
1782         /* pull the interesting pieces out */
1783
1784         rc = process_dynamic(mp, dyndata, dstrdata);
1785
1786         kobj_free(dyndata, dshp->sh_size);
1787         kobj_free(dstrdata, dstrp->sh_size);
1788
1789         return (rc);
1790 }
1791
1792 void
1793 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1794 {
1795         if (!standalone) {
1796                 if (mp->ctfdata != NULL) {
1797                         if (vmem_contains(ctf_arena, mp->ctfdata,
1798                             mp->ctfsize)) {
1799                                 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1800                         } else {
1801                                 kobj_free(mp->ctfdata, mp->ctfsize);
1802                         }
1803                 }
1804         }
1805
1806         /*
1807          * The order is very important here.  We need to make sure that
1808          * consumers, at any given instant, see a consistent state.  We'd
1809          * rather they see no CTF data than the address of one buffer and the
1810          * size of another.
1811          */
1812         mp->ctfdata = NULL;
1813         membar_producer();
1814         mp->ctfsize = size;
1815         mp->ctfdata = data;
1816         membar_producer();
1817 }
1818
1819 int
1820 kobj_load_module(struct modctl *modp, int use_path)
1821 {
1822         char *filename = modp->mod_filename;
1823         char *modname = modp->mod_modname;
1824         int i;
1825         int n;
1826         struct _buf *file;
1827         struct module *mp = NULL;
1828
1829         mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1830
1831         /*
1832          * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1833          * kmdb contains a bunch of symbols with well-known names, symbols
1834          * which will mask the real versions, thus causing no end of trouble
1835          * for mdb.
1836          */
1837         if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1838                 mp->flags |= KOBJ_NOKSYMS;
1839
1840         file = kobj_open_path(filename, use_path);
1841         if (file == (struct _buf *)-1) {
1842                 if (file == (struct _buf *)-1) {
1843                         kobj_free(mp, sizeof (*mp));
1844                         goto bad;
1845                 }
1846         }
1847
1848         mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1849         (void) strcpy(mp->filename, file->_name);
1850
1851         if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1852                 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1853                     modname);
1854                 kobj_free(mp->filename, strlen(file->_name) + 1);
1855                 kobj_free(mp, sizeof (*mp));
1856                 goto bad;
1857         }
1858         for (i = 0; i < SELFMAG; i++) {
1859                 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1860                         if (_moddebug & MODDEBUG_ERRMSG)
1861                                 _kobj_printf(ops, "%s not an elf module\n",
1862                                     modname);
1863                         kobj_free(mp->filename, strlen(file->_name) + 1);
1864                         kobj_free(mp, sizeof (*mp));
1865                         goto bad;
1866                 }
1867         }
1868         /*
1869          * It's ELF, but is it our ISA?  Interpreting the header
1870          * from a file for a byte-swapped ISA could cause a huge
1871          * and unsatisfiable value to be passed to kobj_alloc below
1872          * and therefore hang booting.
1873          */
1874         if (!elf_mach_ok(&mp->hdr)) {
1875                 if (_moddebug & MODDEBUG_ERRMSG)
1876                         _kobj_printf(ops, "%s not an elf module for this ISA\n",
1877                             modname);
1878                 kobj_free(mp->filename, strlen(file->_name) + 1);
1879                 kobj_free(mp, sizeof (*mp));
1880                 goto bad;
1881         }
1882
1883         /*
1884          * All modules, save for unix, should be relocatable (as opposed to
1885          * dynamic).  Dynamic modules come with PLTs and GOTs, which can't
1886          * currently be processed by krtld.
1887          */
1888         if (mp->hdr.e_type != ET_REL) {
1889                 if (_moddebug & MODDEBUG_ERRMSG)
1890                         _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1891                             "module\n", modname);
1892                 kobj_free(mp->filename, strlen(file->_name) + 1);
1893                 kobj_free(mp, sizeof (*mp));
1894                 goto bad;
1895         }
1896
1897         n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1898         mp->shdrs = kobj_alloc(n, KM_WAIT);
1899
1900         if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1901                 _kobj_printf(ops, "kobj_load_module: %s error reading "
1902                     "section headers\n", modname);
1903                 kobj_free(mp->shdrs, n);
1904                 kobj_free(mp->filename, strlen(file->_name) + 1);
1905                 kobj_free(mp, sizeof (*mp));
1906                 goto bad;
1907         }
1908
1909         kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1910         module_assign(modp, mp);
1911
1912         /* read in sections */
1913         if (get_progbits(mp, file) < 0) {
1914                 _kobj_printf(ops, "%s error reading sections\n", modname);
1915                 goto bad;
1916         }
1917
1918         if (do_dynamic(mp, file) < 0) {
1919                 _kobj_printf(ops, "%s error reading dynamic section\n",
1920                     modname);
1921                 goto bad;
1922         }
1923
1924         modp->mod_text = mp->text;
1925         modp->mod_text_size = mp->text_size;
1926
1927         /* read in symbols; adjust values for each section's real address */
1928         if (get_syms(mp, file) < 0) {
1929                 _kobj_printf(ops, "%s error reading symbols\n",
1930                     modname);
1931                 goto bad;
1932         }
1933
1934         /*
1935          * If we didn't dependency information from the dynamic section, look
1936          * for it the old-fashioned way.
1937          */
1938         if (mp->depends_on == NULL)
1939                 mp->depends_on = depends_on(mp);
1940
1941         if (get_ctf(mp, file) < 0) {
1942                 _kobj_printf(ops, "%s debug information will not "
1943                     "be available\n", modname);
1944         }
1945
1946         /* primary kernel modules do not have a signature section */
1947         if (!(mp->flags & KOBJ_PRIM))
1948                 get_signature(mp, file);
1949
1950 #ifdef  KOBJ_DEBUG
1951         if (kobj_debug & D_LOADING) {
1952                 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
1953                 _kobj_printf(ops, "\ttext:0x%p", mp->text);
1954                 _kobj_printf(ops, " size: 0x%x\n", mp->text_size);
1955                 _kobj_printf(ops, "\tdata:0x%p", mp->data);
1956                 _kobj_printf(ops, " dsize: 0x%x\n", mp->data_size);
1957         }
1958 #endif /* KOBJ_DEBUG */
1959
1960         /*
1961          * For primary kernel modules, we defer
1962          * symbol resolution and relocation until
1963          * all primary objects have been loaded.
1964          */
1965         if (!standalone) {
1966                 int ddrval, dcrval;
1967                 char *dependent_modname;
1968                 /* load all dependents */
1969                 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
1970                 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
1971
1972                 /*
1973                  * resolve undefined and common symbols,
1974                  * also allocates common space
1975                  */
1976                 if ((dcrval = do_common(mp)) < 0) {
1977                         switch (dcrval) {
1978                         case DOSYM_UNSAFE:
1979                                 _kobj_printf(ops, "WARNING: mod_load: "
1980                                     "MT-unsafe module '%s' rejected\n",
1981                                     modname);
1982                                 break;
1983                         case DOSYM_UNDEF:
1984                                 _kobj_printf(ops, "WARNING: mod_load: "
1985                                     "cannot load module '%s'\n",
1986                                     modname);
1987                                 if (ddrval == -1) {
1988                                         _kobj_printf(ops, "WARNING: %s: ",
1989                                             modname);
1990                                         _kobj_printf(ops,
1991                                             "unable to resolve dependency, "
1992                                             "module '%s' not found\n",
1993                                             dependent_modname);
1994                                 }
1995                                 break;
1996                         }
1997                 }
1998                 kobj_free(dependent_modname, MODMAXNAMELEN);
1999                 if (dcrval < 0)
2000                         goto bad;
2001
2002                 /* process relocation tables */
2003                 if (do_relocations(mp) < 0) {
2004                         _kobj_printf(ops, "%s error doing relocations\n",
2005                             modname);
2006                         goto bad;
2007                 }
2008
2009                 if (mp->destination) {
2010                         off_t   off = (uintptr_t)mp->destination & PAGEOFFSET;
2011                         caddr_t base = (caddr_t)mp->destination - off;
2012                         size_t  size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2013
2014                         hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2015                         vmem_free(heap_arena, base, size);
2016                 }
2017
2018                 /* sync_instruction_memory */
2019                 kobj_sync_instruction_memory(mp->text, mp->text_size);
2020                 kobj_export_module(mp);
2021                 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2022         }
2023         kobj_close_file(file);
2024         return (0);
2025 bad:
2026         if (file != (struct _buf *)-1)
2027                 kobj_close_file(file);
2028         if (modp->mod_mp != NULL)
2029                 free_module_data(modp->mod_mp);
2030
2031         module_assign(modp, NULL);
2032         return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2033 }
2034
2035 int
2036 kobj_load_primary_module(struct modctl *modp)
2037 {
2038         struct modctl *dep;
2039         struct module *mp;
2040
2041         if (kobj_load_module(modp, 0) != 0)
2042                 return (-1);
2043
2044         mp = modp->mod_mp;
2045         mp->flags |= KOBJ_PRIM;
2046
2047         /* Bind new module to its dependents */
2048         if (mp->depends_on != NULL && (dep =
2049             mod_already_loaded(mp->depends_on)) == NULL) {
2050 #ifdef  KOBJ_DEBUG
2051                 if (kobj_debug & D_DEBUG) {
2052                         _kobj_printf(ops, "krtld: failed to resolve deps "
2053                             "for primary %s\n", modp->mod_modname);
2054                 }
2055 #endif
2056                 return (-1);
2057         }
2058
2059         add_dependent(mp, dep->mod_mp);
2060
2061         /*
2062          * Relocate it.  This module may not be part of a link map, so we
2063          * can't use bind_primary.
2064          */
2065         if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2066             do_relocations(mp) < 0) {
2067 #ifdef  KOBJ_DEBUG
2068                 if (kobj_debug & D_DEBUG) {
2069                         _kobj_printf(ops, "krtld: failed to relocate "
2070                             "primary %s\n", modp->mod_modname);
2071                 }
2072 #endif
2073                 return (-1);
2074         }
2075
2076         return (0);
2077 }
2078
2079 static void
2080 module_assign(struct modctl *cp, struct module *mp)
2081 {
2082         if (standalone) {
2083                 cp->mod_mp = mp;
2084                 return;
2085         }
2086         mutex_enter(&mod_lock);
2087         cp->mod_mp = mp;
2088         cp->mod_gencount++;
2089         mutex_exit(&mod_lock);
2090 }
2091
2092 void
2093 kobj_unload_module(struct modctl *modp)
2094 {
2095         struct module *mp = modp->mod_mp;
2096
2097         if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2098                 _kobj_printf(ops, "text for %s ", mp->filename);
2099                 _kobj_printf(ops, "was at %p\n", mp->text);
2100                 mp->text = NULL;        /* don't actually free it */
2101         }
2102
2103         kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2104
2105         /*
2106          * Null out mod_mp first, so consumers (debuggers) know not to look
2107          * at the module structure any more.
2108          */
2109         mutex_enter(&mod_lock);
2110         modp->mod_mp = NULL;
2111         mutex_exit(&mod_lock);
2112
2113         kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2114         free_module_data(mp);
2115 }
2116
2117 static void
2118 free_module_data(struct module *mp)
2119 {
2120         struct module_list *lp, *tmp;
2121         int ksyms_exported = 0;
2122
2123         lp = mp->head;
2124         while (lp) {
2125                 tmp = lp;
2126                 lp = lp->next;
2127                 kobj_free((char *)tmp, sizeof (*tmp));
2128         }
2129
2130         rw_enter(&ksyms_lock, RW_WRITER);
2131         if (mp->symspace) {
2132                 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2133                         vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2134                         ksyms_exported = 1;
2135                 } else {
2136                         if (mp->flags & KOBJ_NOKSYMS)
2137                                 ksyms_exported = 1;
2138                         kobj_free(mp->symspace, mp->symsize);
2139                 }
2140         }
2141         rw_exit(&ksyms_lock);
2142
2143         if (mp->ctfdata) {
2144                 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2145                         vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2146                 else
2147                         kobj_free(mp->ctfdata, mp->ctfsize);
2148         }
2149
2150         if (mp->sigdata)
2151                 kobj_free(mp->sigdata, mp->sigsize);
2152
2153         /*
2154          * We did not get far enough into kobj_export_ksyms() to free allocated
2155          * buffers because we encounted error conditions. Free the buffers.
2156          */
2157         if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2158                 uint_t shn;
2159                 Shdr *shp;
2160
2161                 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2162                         shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2163                         switch (shp->sh_type) {
2164                         case SHT_RELA:
2165                         case SHT_REL:
2166                                 if (shp->sh_addr != 0)
2167                                         kobj_free((void *)shp->sh_addr,
2168                                             shp->sh_size);
2169                                 break;
2170                         }
2171                 }
2172 err_free_done:
2173                 if (!(mp->flags & KOBJ_PRIM)) {
2174                         kobj_free(mp->shdrs,
2175                             mp->hdr.e_shentsize * mp->hdr.e_shnum);
2176                 }
2177         }
2178
2179         if (mp->bss)
2180                 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2181
2182         if (mp->fbt_tab)
2183                 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2184
2185         if (mp->textwin_base)
2186                 kobj_textwin_free(mp);
2187
2188         if (mp->sdt_probes != NULL) {
2189                 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2190
2191                 while (sdp != NULL) {
2192                         next = sdp->sdpd_next;
2193                         kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2194                         kobj_free(sdp, sizeof (sdt_probedesc_t));
2195                         sdp = next;
2196                 }
2197         }
2198
2199         if (mp->sdt_tab)
2200                 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2201         if (mp->text)
2202                 vmem_free(text_arena, mp->text, mp->text_size);
2203         if (mp->data)
2204                 vmem_free(data_arena, mp->data, mp->data_size);
2205         if (mp->depends_on)
2206                 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2207         if (mp->filename)
2208                 kobj_free(mp->filename, strlen(mp->filename)+1);
2209
2210         kobj_free((char *)mp, sizeof (*mp));
2211 }
2212
2213 static int
2214 get_progbits(struct module *mp, struct _buf *file)
2215 {
2216         struct proginfo *tp, *dp, *sdp;
2217         Shdr *shp;
2218         reloc_dest_t dest = NULL;
2219         uintptr_t bits_ptr;
2220         uintptr_t text = 0, data, textptr;
2221         uint_t shn;
2222         int err = -1;
2223
2224         tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2225         dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2226         sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2227         /*
2228          * loop through sections to find out how much space we need
2229          * for text, data, (also bss that is already assigned)
2230          */
2231         if (get_progbits_size(mp, tp, dp, sdp) < 0)
2232                 goto done;
2233
2234         mp->text_size = tp->size;
2235         mp->data_size = dp->size;
2236
2237         if (standalone) {
2238                 caddr_t limit = _data;
2239
2240                 if (lg_pagesize && _text + lg_pagesize < limit)
2241                         limit = _text + lg_pagesize;
2242
2243                 mp->text = kobj_segbrk(&_etext, mp->text_size,
2244                     tp->align, limit);
2245                 /*
2246                  * If we can't grow the text segment, try the
2247                  * data segment before failing.
2248                  */
2249                 if (mp->text == NULL) {
2250                         mp->text = kobj_segbrk(&_edata, mp->text_size,
2251                             tp->align, 0);
2252                 }
2253
2254                 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2255
2256                 if (mp->text == NULL || mp->data == NULL)
2257                         goto done;
2258
2259         } else {
2260                 if (text_arena == NULL)
2261                         kobj_vmem_init(&text_arena, &data_arena);
2262
2263                 /*
2264                  * some architectures may want to load the module on a
2265                  * page that is currently read only. It may not be
2266                  * possible for those architectures to remap their page
2267                  * on the fly. So we provide a facility for them to hang
2268                  * a private hook where the memory they assign the module
2269                  * is not the actual place where the module loads.
2270                  *
2271                  * In this case there are two addresses that deal with the
2272                  * modload.
2273                  * 1) the final destination of the module
2274                  * 2) the address that is used to view the newly
2275                  * loaded module until all the relocations relative to 1
2276                  * above are completed.
2277                  *
2278                  * That is what dest is used for below.
2279                  */
2280                 mp->text_size += tp->align;
2281                 mp->data_size += dp->align;
2282
2283                 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2284
2285                 /*
2286                  * a remap is taking place. Align the text ptr relative
2287                  * to the secondary mapping. That is where the bits will
2288                  * be read in.
2289                  */
2290                 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2291                     mp->text, mp->text_size)) {
2292                         off_t   off = (uintptr_t)mp->text & PAGEOFFSET;
2293                         size_t  size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2294                         caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2295                         caddr_t orig = mp->text - off;
2296                         pgcnt_t pages = size / PAGESIZE;
2297
2298                         dest = (reloc_dest_t)(map + off);
2299                         text = ALIGN((uintptr_t)dest, tp->align);
2300
2301                         while (pages--) {
2302                                 hat_devload(kas.a_hat, map, PAGESIZE,
2303                                     hat_getpfnum(kas.a_hat, orig),
2304                                     PROT_READ | PROT_WRITE | PROT_EXEC,
2305                                     HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2306                                 map += PAGESIZE;
2307                                 orig += PAGESIZE;
2308                         }
2309                         /*
2310                          * Since we set up a non-cacheable mapping, we need
2311                          * to flush any old entries in the cache that might
2312                          * be left around from the read-only mapping.
2313                          */
2314                         dcache_flushall();
2315                 }
2316                 if (mp->data_size)
2317                         mp->data = vmem_alloc(data_arena, mp->data_size,
2318                             VM_SLEEP | VM_BESTFIT);
2319         }
2320         textptr = (uintptr_t)mp->text;
2321         textptr = ALIGN(textptr, tp->align);
2322         mp->destination = dest;
2323
2324         /*
2325          * This is the case where a remap is not being done.
2326          */
2327         if (text == 0)
2328                 text = ALIGN((uintptr_t)mp->text, tp->align);
2329         data = ALIGN((uintptr_t)mp->data, dp->align);
2330
2331         /* now loop though sections assigning addresses and loading the data */
2332         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2333                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2334                 if (!(shp->sh_flags & SHF_ALLOC))
2335                         continue;
2336
2337                 if ((shp->sh_flags & SHF_WRITE) == 0)
2338                         bits_ptr = text;
2339                 else
2340                         bits_ptr = data;
2341
2342                 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2343
2344                 if (shp->sh_type == SHT_NOBITS) {
2345                         /*
2346                          * Zero bss.
2347                          */
2348                         bzero((caddr_t)bits_ptr, shp->sh_size);
2349                         shp->sh_type = SHT_PROGBITS;
2350                 } else {
2351                         if (kobj_read_file(file, (char *)bits_ptr,
2352                             shp->sh_size, shp->sh_offset) < 0)
2353                                 goto done;
2354                 }
2355
2356                 if (shp->sh_flags & SHF_WRITE) {
2357                         shp->sh_addr = bits_ptr;
2358                 } else {
2359                         textptr = ALIGN(textptr, shp->sh_addralign);
2360                         shp->sh_addr = textptr;
2361                         textptr += shp->sh_size;
2362                 }
2363
2364                 bits_ptr += shp->sh_size;
2365                 if ((shp->sh_flags & SHF_WRITE) == 0)
2366                         text = bits_ptr;
2367                 else
2368                         data = bits_ptr;
2369         }
2370
2371         err = 0;
2372 done:
2373         /*
2374          * Free and mark as freed the section headers here so that
2375          * free_module_data() does not have to worry about this buffer.
2376          *
2377          * This buffer is freed here because one of the possible reasons
2378          * for error is a section with non-zero sh_addr and in that case
2379          * free_module_data() would have no way of recognizing that this
2380          * buffer was unallocated.
2381          */
2382         if (err != 0) {
2383                 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2384                 mp->shdrs = NULL;
2385         }
2386
2387         (void) kobj_free(tp, sizeof (struct proginfo));
2388         (void) kobj_free(dp, sizeof (struct proginfo));
2389         (void) kobj_free(sdp, sizeof (struct proginfo));
2390
2391         return (err);
2392 }
2393
2394 /*
2395  * Go through suppress_sym_list to see if "multiply defined"
2396  * warning of this symbol should be suppressed.  Return 1 if
2397  * warning should be suppressed, 0 otherwise.
2398  */
2399 static int
2400 kobj_suppress_warning(char *symname)
2401 {
2402         int     i;
2403
2404         for (i = 0; suppress_sym_list[i] != NULL; i++) {
2405                 if (strcmp(suppress_sym_list[i], symname) == 0)
2406                         return (1);
2407         }
2408
2409         return (0);
2410 }
2411
2412 static int
2413 get_syms(struct module *mp, struct _buf *file)
2414 {
2415         uint_t          shn;
2416         Shdr    *shp;
2417         uint_t          i;
2418         Sym     *sp, *ksp;
2419         char            *symname;
2420         int             dosymtab = 0;
2421
2422         /*
2423          * Find the interesting sections.
2424          */
2425         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2426                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2427                 switch (shp->sh_type) {
2428                 case SHT_SYMTAB:
2429                         mp->symtbl_section = shn;
2430                         mp->symhdr = shp;
2431                         dosymtab++;
2432                         break;
2433
2434                 case SHT_RELA:
2435                 case SHT_REL:
2436                         /*
2437                          * Already loaded.
2438                          */
2439                         if (shp->sh_addr)
2440                                 continue;
2441
2442                         /* KM_TMP since kobj_free'd in do_relocations */
2443                         shp->sh_addr = (Addr)
2444                             kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2445
2446                         if (kobj_read_file(file, (char *)shp->sh_addr,
2447                             shp->sh_size, shp->sh_offset) < 0) {
2448                                 _kobj_printf(ops, "krtld: get_syms: %s, ",
2449                                     mp->filename);
2450                                 _kobj_printf(ops, "error reading section %d\n",
2451                                     shn);
2452                                 return (-1);
2453                         }
2454                         break;
2455                 }
2456         }
2457
2458         /*
2459          * This is true for a stripped executable.  In the case of
2460          * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2461          * and since that symbol information is still present everything
2462          * is just fine.
2463          */
2464         if (!dosymtab) {
2465                 if (mp->flags & KOBJ_EXEC)
2466                         return (0);
2467                 _kobj_printf(ops, "krtld: get_syms: %s ",
2468                     mp->filename);
2469                 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2470                 return (-1);
2471         }
2472
2473         /*
2474          * get the associated string table header
2475          */
2476         if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2477                 return (-1);
2478         mp->strhdr = (Shdr *)
2479             (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2480
2481         mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2482         mp->hashsize = kobj_gethashsize(mp->nsyms);
2483
2484         /*
2485          * Allocate space for the symbol table, buckets, chains, and strings.
2486          */
2487         mp->symsize = mp->symhdr->sh_size +
2488             (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2489         mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2490
2491         mp->symtbl = mp->symspace;
2492         mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2493         mp->chains = mp->buckets + mp->hashsize;
2494         mp->strings = (char *)(mp->chains + mp->nsyms);
2495
2496         if (kobj_read_file(file, mp->symtbl,
2497             mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2498             kobj_read_file(file, mp->strings,
2499             mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2500                 return (-1);
2501
2502         /*
2503          * loop through the symbol table adjusting values to account
2504          * for where each section got loaded into memory.  Also
2505          * fill in the hash table.
2506          */
2507         for (i = 1; i < mp->nsyms; i++) {
2508                 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2509                 if (sp->st_shndx < SHN_LORESERVE) {
2510                         if (sp->st_shndx >= mp->hdr.e_shnum) {
2511                                 _kobj_printf(ops, "%s bad shndx ",
2512                                     file->_name);
2513                                 _kobj_printf(ops, "in symbol %d\n", i);
2514                                 return (-1);
2515                         }
2516                         shp = (Shdr *)
2517                             (mp->shdrs +
2518                             sp->st_shndx * mp->hdr.e_shentsize);
2519                         if (!(mp->flags & KOBJ_EXEC))
2520                                 sp->st_value += shp->sh_addr;
2521                 }
2522
2523                 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2524                         continue;
2525                 if (sp->st_name >= mp->strhdr->sh_size)
2526                         return (-1);
2527
2528                 symname = mp->strings + sp->st_name;
2529
2530                 if (!(mp->flags & KOBJ_EXEC) &&
2531                     ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2532                         ksp = kobj_lookup_all(mp, symname, 0);
2533
2534                         if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2535                             !kobj_suppress_warning(symname) &&
2536                             sp->st_shndx != SHN_UNDEF &&
2537                             sp->st_shndx != SHN_COMMON &&
2538                             ksp->st_shndx != SHN_UNDEF &&
2539                             ksp->st_shndx != SHN_COMMON) {
2540                                 /*
2541                                  * Unless this symbol is a stub, it's multiply
2542                                  * defined.  Multiply-defined symbols are
2543                                  * usually bad, but some objects (kmdb) have
2544                                  * a legitimate need to have their own
2545                                  * copies of common functions.
2546                                  */
2547                                 if ((standalone ||
2548                                     ksp->st_value < (uintptr_t)stubs_base ||
2549                                     ksp->st_value >= (uintptr_t)stubs_end) &&
2550                                     !(mp->flags & KOBJ_IGNMULDEF)) {
2551                                         _kobj_printf(ops,
2552                                             "%s symbol ", file->_name);
2553                                         _kobj_printf(ops,
2554                                             "%s multiply defined\n", symname);
2555                                 }
2556                         }
2557                 }
2558
2559                 sym_insert(mp, symname, i);
2560         }
2561
2562         return (0);
2563 }
2564
2565 static int
2566 get_ctf(struct module *mp, struct _buf *file)
2567 {
2568         char *shstrtab, *ctfdata;
2569         size_t shstrlen;
2570         Shdr *shp;
2571         uint_t i;
2572
2573         if (_moddebug & MODDEBUG_NOCTF)
2574                 return (0); /* do not attempt to even load CTF data */
2575
2576         if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2577                 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2578                     mp->filename);
2579                 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2580                     mp->hdr.e_shstrndx);
2581                 return (-1);
2582         }
2583
2584         shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2585         shstrlen = shp->sh_size;
2586         shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2587
2588         if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2589                 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2590                     mp->filename);
2591                 _kobj_printf(ops, "error reading section %u\n",
2592                     mp->hdr.e_shstrndx);
2593                 kobj_free(shstrtab, shstrlen);
2594                 return (-1);
2595         }
2596
2597         for (i = 0; i < mp->hdr.e_shnum; i++) {
2598                 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2599
2600                 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2601                     strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2602                         ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2603
2604                         if (kobj_read_file(file, ctfdata, shp->sh_size,
2605                             shp->sh_offset) < 0) {
2606                                 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2607                                     "reading .SUNW_ctf data\n", mp->filename);
2608                                 kobj_free(ctfdata, shp->sh_size);
2609                                 kobj_free(shstrtab, shstrlen);
2610                                 return (-1);
2611                         }
2612
2613                         mp->ctfdata = ctfdata;
2614                         mp->ctfsize = shp->sh_size;
2615                         break;
2616                 }
2617         }
2618
2619         kobj_free(shstrtab, shstrlen);
2620         return (0);
2621 }
2622
2623 #define SHA1_DIGEST_LENGTH      20      /* SHA1 digest length in bytes */
2624
2625 /*
2626  * Return the hash of the ELF sections that are memory resident.
2627  * i.e. text and data.  We skip a SHT_NOBITS section since it occupies
2628  * no space in the file. We use SHA1 here since libelfsign uses
2629  * it and both places need to use the same algorithm.
2630  */
2631 static void
2632 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2633 {
2634         uint_t shn;
2635         Shdr *shp;
2636         SHA1_CTX ctx;
2637
2638         SHA1Init(&ctx);
2639
2640         for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2641                 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2642                 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2643                         continue;
2644
2645                 /*
2646                  * The check should ideally be shp->sh_type == SHT_NOBITS.
2647                  * However, we can't do that check here as get_progbits()
2648                  * resets the type.
2649                  */
2650                 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2651                         continue;
2652 #ifdef  KOBJ_DEBUG
2653                 if (kobj_debug & D_DEBUG)
2654                         _kobj_printf(ops,
2655                             "krtld: crypto_es_hash: updating hash with"
2656                             " %s data size=%d\n", shstrtab + shp->sh_name,
2657                             shp->sh_size);
2658 #endif
2659                 ASSERT(shp->sh_addr != (uintptr_t)NULL);
2660                 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2661         }
2662
2663         SHA1Final((uchar_t *)hash, &ctx);
2664 }
2665
2666 /*
2667  * Get the .SUNW_signature section for the module, it it exists.
2668  *
2669  * This section exists only for crypto modules. None of the
2670  * primary modules have this section currently.
2671  */
2672 static void
2673 get_signature(struct module *mp, struct _buf *file)
2674 {
2675         char *shstrtab, *sigdata = NULL;
2676         size_t shstrlen;
2677         Shdr *shp;
2678         uint_t i;
2679
2680         if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2681                 _kobj_printf(ops, "krtld: get_signature: %s, ",
2682                     mp->filename);
2683                 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2684                     mp->hdr.e_shstrndx);
2685                 return;
2686         }
2687
2688         shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2689         shstrlen = shp->sh_size;
2690         shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2691
2692         if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2693                 _kobj_printf(ops, "krtld: get_signature: %s, ",
2694                     mp->filename);
2695                 _kobj_printf(ops, "error reading section %u\n",
2696                     mp->hdr.e_shstrndx);
2697                 kobj_free(shstrtab, shstrlen);
2698                 return;
2699         }
2700
2701         for (i = 0; i < mp->hdr.e_shnum; i++) {
2702                 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2703                 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2704                     strcmp(shstrtab + shp->sh_name,
2705                     ELF_SIGNATURE_SECTION) == 0) {
2706                         filesig_vers_t filesig_version;
2707                         size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2708                         sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2709
2710                         if (kobj_read_file(file, sigdata, shp->sh_size,
2711                             shp->sh_offset) < 0) {
2712                                 _kobj_printf(ops, "krtld: get_signature: %s,"
2713                                     " error reading .SUNW_signature data\n",
2714                                     mp->filename);
2715                                 kobj_free(sigdata, sigsize);
2716                                 kobj_free(shstrtab, shstrlen);
2717                                 return;
2718                         }
2719                         filesig_version = ((struct filesignatures *)sigdata)->
2720                             filesig_sig.filesig_version;
2721                         if (!(filesig_version == FILESIG_VERSION1 ||
2722                             filesig_version == FILESIG_VERSION3)) {
2723                                 /* skip versions we don't understand */
2724                                 kobj_free(sigdata, sigsize);
2725                                 kobj_free(shstrtab, shstrlen);
2726                                 return;
2727                         }
2728
2729                         mp->sigdata = sigdata;
2730                         mp->sigsize = sigsize;
2731                         break;
2732                 }
2733         }
2734
2735         if (sigdata != NULL) {
2736                 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2737         }
2738
2739         kobj_free(shstrtab, shstrlen);
2740 }
2741
2742 static void
2743 add_dependent(struct module *mp, struct module *dep)
2744 {
2745         struct module_list *lp;
2746
2747         for (lp = mp->head; lp; lp = lp->next) {
2748                 if (lp->mp == dep)
2749                         return; /* already on the list */
2750         }
2751
2752         if (lp == NULL) {
2753                 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2754
2755                 lp->mp = dep;
2756                 lp->next = NULL;
2757                 if (mp->tail)
2758                         mp->tail->next = lp;
2759                 else
2760                         mp->head = lp;
2761                 mp->tail = lp;
2762         }
2763 }
2764
2765 static int
2766 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2767 {
2768         struct module *mp;
2769         struct modctl *req;
2770         char *d, *p, *q;
2771         int c;
2772         char *err_modname = NULL;
2773
2774         mp = modp->mod_mp;
2775
2776         if ((p = mp->depends_on) == NULL)
2777                 return (0);
2778
2779         for (;;) {
2780                 /*
2781                  * Skip space.
2782                  */
2783                 while (*p && (*p == ' ' || *p == '\t'))
2784                         p++;
2785                 /*
2786                  * Get module name.
2787                  */
2788                 d = p;
2789                 q = modname;
2790                 c = 0;
2791                 while (*p && *p != ' ' && *p != '\t') {
2792                         if (c < modnamelen - 1) {
2793                                 *q++ = *p;
2794                                 c++;
2795                         }
2796                         p++;
2797                 }
2798
2799                 if (q == modname)
2800                         break;
2801
2802                 if (c == modnamelen - 1) {
2803                         char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2804
2805                         (void) strncpy(dep, d,  p - d + 1);
2806                         dep[p - d] = '\0';
2807
2808                         _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2809                         _kobj_printf(ops, "'%s' too long ", dep);
2810                         _kobj_printf(ops, "(max %d chars)\n", modnamelen);
2811
2812                         kobj_free(dep, p - d + 1);
2813
2814                         return (-1);
2815                 }
2816
2817                 *q = '\0';
2818                 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2819 #ifndef KOBJ_DEBUG
2820                         if (_moddebug & MODDEBUG_LOADMSG) {
2821 #endif  /* KOBJ_DEBUG */
2822                                 _kobj_printf(ops,
2823                                     "%s: unable to resolve dependency, ",
2824                                     modp->mod_modname);
2825                                 _kobj_printf(ops, "cannot load module '%s'\n",
2826                                     modname);
2827 #ifndef KOBJ_DEBUG
2828                         }
2829 #endif  /* KOBJ_DEBUG */
2830                         if (err_modname == NULL) {
2831                                 /*
2832                                  * This must be the same size as the modname
2833                                  * one.
2834                                  */
2835                                 err_modname = kobj_zalloc(MODMAXNAMELEN,
2836                                     KM_WAIT);
2837
2838                                 /*
2839                                  * We can use strcpy() here without fearing
2840                                  * the NULL terminator because the size of
2841                                  * err_modname is the same as one of modname,
2842                                  * and it's filled with zeros.
2843                                  */
2844                                 (void) strcpy(err_modname, modname);
2845                         }
2846                         continue;
2847                 }
2848
2849                 add_dependent(mp, req->mod_mp);
2850                 mod_release_mod(req);
2851
2852         }
2853
2854         if (err_modname != NULL) {
2855                 /*
2856                  * Copy the first module name where you detect an error to keep
2857                  * its behavior the same as before.
2858                  * This way keeps minimizing the memory use for error
2859                  * modules, and this might be important at boot time because
2860                  * the memory usage is a crucial factor for booting in most
2861                  * cases. You can expect more verbose messages when using
2862                  * a debug kernel or setting a bit in moddebug.
2863                  */
2864                 bzero(modname, MODMAXNAMELEN);
2865                 (void) strcpy(modname, err_modname);
2866                 kobj_free(err_modname, MODMAXNAMELEN);
2867                 return (-1);
2868         }
2869
2870         return (0);
2871 }
2872
2873 static int
2874 do_common(struct module *mp)
2875 {
2876         int err;
2877
2878         /*
2879          * first time through, assign all symbols defined in other
2880          * modules, and count up how much common space will be needed
2881          * (bss_size and bss_align)
2882          */
2883         if ((err = do_symbols(mp, 0)) < 0)
2884                 return (err);
2885         /*
2886          * increase bss_size by the maximum delta that could be
2887          * computed by the ALIGN below
2888          */
2889         mp->bss_size += mp->bss_align;
2890         if (mp->bss_size) {
2891                 if (standalone)
2892                         mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2893                             MINALIGN, 0);
2894                 else
2895                         mp->bss = (uintptr_t)vmem_alloc(data_arena,
2896                             mp->bss_size, VM_SLEEP | VM_BESTFIT);
2897                 bzero((void *)mp->bss, mp->bss_size);
2898                 /* now assign addresses to all common symbols */
2899                 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2900                         return (err);
2901         }
2902         return (0);
2903 }
2904
2905 static int
2906 do_symbols(struct module *mp, Elf64_Addr bss_base)
2907 {
2908         int bss_align;
2909         uintptr_t bss_ptr;
2910         int err;
2911         int i;
2912         Sym *sp, *sp1;
2913         char *name;
2914         int assign;
2915         int resolved = 1;
2916
2917         /*
2918          * Nothing left to do (optimization).
2919          */
2920         if (mp->flags & KOBJ_RESOLVED)
2921                 return (0);
2922
2923         assign = (bss_base) ? 1 : 0;
2924         bss_ptr = bss_base;
2925         bss_align = 0;
2926         err = 0;
2927
2928         for (i = 1; i < mp->nsyms; i++) {
2929                 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
2930                 /*
2931                  * we know that st_name is in bounds, since get_sections
2932                  * has already checked all of the symbols
2933                  */
2934                 name = mp->strings + sp->st_name;
2935                 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
2936                         continue;
2937                 /*
2938                  * TLS symbols are ignored in the kernel
2939                  */
2940                 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
2941                         _kobj_printf(ops, "%s: TLS symbol ",
2942                             mp->filename);
2943                         _kobj_printf(ops, "not supported '%s'\n",
2944                             name);
2945                         err = DOSYM_UNDEF;
2946                         continue;
2947                 }
2948
2949                 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
2950                         if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
2951                                 sp->st_shndx = SHN_ABS;
2952                                 sp->st_value = sp1->st_value;
2953                                 continue;
2954                         }
2955                 }
2956
2957                 if (sp->st_shndx == SHN_UNDEF) {
2958                         resolved = 0;
2959
2960                         if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
2961                                 continue;
2962
2963                         /*
2964                          * If it's not a weak reference and it's
2965                          * not a primary object, it's an error.
2966                          * (Primary objects may take more than
2967                          * one pass to resolve)
2968                          */
2969                         if (!(mp->flags & KOBJ_PRIM) &&
2970                             ELF_ST_BIND(sp->st_info) != STB_WEAK) {
2971                                 _kobj_printf(ops, "%s: undefined symbol",
2972                                     mp->filename);
2973                                 _kobj_printf(ops, " '%s'\n", name);
2974                                 /*
2975                                  * Try to determine whether this symbol
2976                                  * represents a dependency on obsolete
2977                                  * unsafe driver support.  This is just
2978                                  * to make the warning more informative.
2979                                  */
2980                                 if (strcmp(name, "sleep") == 0 ||
2981                                     strcmp(name, "unsleep") == 0 ||
2982                                     strcmp(name, "wakeup") == 0 ||
2983                                     strcmp(name, "bsd_compat_ioctl") == 0 ||
2984                                     strcmp(name, "unsafe_driver") == 0 ||
2985                                     strncmp(name, "spl", 3) == 0 ||
2986                                     strncmp(name, "i_ddi_spl", 9) == 0)
2987                                         err = DOSYM_UNSAFE;
2988                                 if (err == 0)
2989                                         err = DOSYM_UNDEF;
2990                         }
2991                         continue;
2992                 }
2993                 /*
2994                  * It's a common symbol - st_value is the
2995                  * required alignment.
2996                  */
2997                 if (sp->st_value > bss_align)
2998                         bss_align = sp->st_value;
2999                 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3000                 if (assign) {
3001                         sp->st_shndx = SHN_ABS;
3002                         sp->st_value = bss_ptr;
3003                 }
3004                 bss_ptr += sp->st_size;
3005         }
3006         if (err)
3007                 return (err);
3008         if (assign == 0 && mp->bss == (uintptr_t)NULL) {
3009                 mp->bss_align = bss_align;
3010                 mp->bss_size = bss_ptr;
3011         } else if (resolved) {
3012                 mp->flags |= KOBJ_RESOLVED;
3013         }
3014
3015         return (0);
3016 }
3017
3018 uint_t
3019 kobj_hash_name(const char *p)
3020 {
3021         uint_t g;
3022         uint_t hval;
3023
3024         hval = 0;
3025         while (*p) {
3026                 hval = (hval << 4) + *p++;
3027                 if ((g = (hval & 0xf0000000)) != 0)
3028                         hval ^= g >> 24;
3029                 hval &= ~g;
3030         }
3031         return (hval);
3032 }
3033
3034 /* look for name in all modules */
3035 uintptr_t
3036 kobj_getsymvalue(char *name, int kernelonly)
3037 {
3038         Sym             *sp;
3039         struct modctl   *modp;
3040         struct module   *mp;
3041         uintptr_t       value = 0;
3042
3043         if ((sp = kobj_lookup_kernel(name)) != NULL)
3044                 return ((uintptr_t)sp->st_value);
3045
3046         if (kernelonly)
3047                 return (0);     /* didn't find it in the kernel so give up */
3048
3049         mutex_enter(&mod_lock);
3050         modp = &modules;
3051         do {
3052                 mp = (struct module *)modp->mod_mp;
3053                 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3054                     (sp = lookup_one(mp, name))) {
3055                         value = (uintptr_t)sp->st_value;
3056                         break;
3057                 }
3058         } while ((modp = modp->mod_next) != &modules);
3059         mutex_exit(&mod_lock);
3060         return (value);
3061 }
3062
3063 /* look for a symbol near value. */
3064 char *
3065 kobj_getsymname(uintptr_t value, ulong_t *offset)
3066 {
3067         char *name = NULL;
3068         struct modctl *modp;
3069
3070         struct modctl_list *lp;
3071         struct module *mp;
3072
3073         /*
3074          * Loop through the primary kernel modules.
3075          */
3076         for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3077                 mp = mod(lp);
3078
3079                 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3080                         return (name);
3081         }
3082
3083         mutex_enter(&mod_lock);
3084         modp = &modules;
3085         do {
3086                 mp = (struct module *)modp->mod_mp;
3087                 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3088                     (name = kobj_searchsym(mp, value, offset)))
3089                         break;
3090         } while ((modp = modp->mod_next) != &modules);
3091         mutex_exit(&mod_lock);
3092         return (name);
3093 }
3094
3095 /* return address of symbol and size */
3096
3097 uintptr_t
3098 kobj_getelfsym(char *name, void *mp, int *size)
3099 {
3100         Sym *sp;
3101
3102         if (mp == NULL)
3103                 sp = kobj_lookup_kernel(name);
3104         else
3105                 sp = lookup_one(mp, name);
3106
3107         if (sp == NULL)
3108                 return (0);
3109
3110         *size = (int)sp->st_size;
3111         return ((uintptr_t)sp->st_value);
3112 }
3113
3114 uintptr_t
3115 kobj_lookup(struct module *mod, const char *name)
3116 {
3117         Sym *sp;
3118
3119         sp = lookup_one(mod, name);
3120
3121         if (sp == NULL)
3122                 return (0);
3123
3124         return ((uintptr_t)sp->st_value);
3125 }
3126
3127 char *
3128 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3129 {
3130         Sym *symtabptr;
3131         char *strtabptr;
3132         int symnum;
3133         Sym *sym;
3134         Sym *cursym;
3135         uintptr_t curval;
3136
3137         *offset = (ulong_t)-1l;         /* assume not found */
3138         cursym  = NULL;
3139
3140         if (kobj_addrcheck(mp, (void *)value) != 0)
3141                 return (NULL);          /* not in this module */
3142
3143         strtabptr  = mp->strings;
3144         symtabptr  = (Sym *)mp->symtbl;
3145
3146         /*
3147          * Scan the module's symbol table for a symbol <= value
3148          */
3149         for (symnum = 1, sym = symtabptr + 1;
3150             symnum < mp->nsyms; symnum++, sym = (Sym *)
3151             ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3152                 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3153                         if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3154                                 continue;
3155                         if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3156                             ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3157                                 continue;
3158                 }
3159
3160                 curval = (uintptr_t)sym->st_value;
3161
3162                 if (curval > value)
3163                         continue;
3164
3165                 /*
3166                  * If one or both are functions...
3167                  */
3168                 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3169                     ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3170                         /* Ignore if the address is out of the bounds */
3171                         if (value - sym->st_value >= sym->st_size)
3172                                 continue;
3173
3174                         if (cursym != NULL &&
3175                             ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3176                                 /* Prefer the function to the non-function */
3177                                 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3178                                         continue;
3179
3180                                 /* Prefer the larger of the two functions */
3181                                 if (sym->st_size <= cursym->st_size)
3182                                         continue;
3183                         }
3184                 } else if (value - curval >= *offset) {
3185                         continue;
3186                 }
3187
3188                 *offset = (ulong_t)(value - curval);
3189                 cursym = sym;
3190         }
3191         if (cursym == NULL)
3192                 return (NULL);
3193
3194         return (strtabptr + cursym->st_name);
3195 }
3196
3197 Sym *
3198 kobj_lookup_all(struct module *mp, char *name, int include_self)
3199 {
3200         Sym *sp;
3201         struct module_list *mlp;
3202         struct modctl_list *clp;
3203         struct module *mmp;
3204
3205         if (include_self && (sp = lookup_one(mp, name)) != NULL)
3206                 return (sp);
3207
3208         for (mlp = mp->head; mlp; mlp = mlp->next) {
3209                 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3210                     ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3211                         return (sp);
3212         }
3213
3214         /*
3215          * Loop through the primary kernel modules.
3216          */
3217         for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3218                 mmp = mod(clp);
3219
3220                 if (mmp == NULL || mp == mmp)
3221                         continue;
3222
3223                 if ((sp = lookup_one(mmp, name)) != NULL &&
3224                     ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3225                         return (sp);
3226         }
3227         return (NULL);
3228 }
3229
3230 Sym *
3231 kobj_lookup_kernel(const char *name)
3232 {
3233         struct modctl_list *lp;
3234         struct module *mp;
3235         Sym *sp;
3236
3237         /*
3238          * Loop through the primary kernel modules.
3239          */
3240         for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3241                 mp = mod(lp);
3242
3243                 if (mp == NULL)
3244                         continue;
3245
3246                 if ((sp = lookup_one(mp, name)) != NULL)
3247                         return (sp);
3248         }
3249         return (NULL);
3250 }
3251
3252 static Sym *
3253 lookup_one(struct module *mp, const char *name)
3254 {
3255         symid_t *ip;
3256         char *name1;
3257         Sym *sp;
3258
3259         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3260             ip = &mp->chains[*ip]) {
3261                 sp = (Sym *)(mp->symtbl +
3262                     mp->symhdr->sh_entsize * *ip);
3263                 name1 = mp->strings + sp->st_name;
3264                 if (strcmp(name, name1) == 0 &&
3265                     ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3266                     sp->st_shndx != SHN_UNDEF &&
3267                     sp->st_shndx != SHN_COMMON)
3268                         return (sp);
3269         }
3270         return (NULL);
3271 }
3272
3273 /*
3274  * Lookup a given symbol pointer in the module's symbol hash.  If the symbol
3275  * is hashed, return the symbol pointer; otherwise return NULL.
3276  */
3277 static Sym *
3278 sym_lookup(struct module *mp, Sym *ksp)
3279 {
3280         char *name = mp->strings + ksp->st_name;
3281         symid_t *ip;
3282         Sym *sp;
3283
3284         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3285             ip = &mp->chains[*ip]) {
3286                 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3287                 if (sp == ksp)
3288                         return (ksp);
3289         }
3290         return (NULL);
3291 }
3292
3293 static void
3294 sym_insert(struct module *mp, char *name, symid_t index)
3295 {
3296         symid_t *ip;
3297
3298 #ifdef KOBJ_DEBUG
3299         if (kobj_debug & D_SYMBOLS) {
3300                 static struct module *lastmp = NULL;
3301                 Sym *sp;
3302                 if (lastmp != mp) {
3303                         _kobj_printf(ops,
3304                             "krtld: symbol entry: file=%s\n",
3305                             mp->filename);
3306                         _kobj_printf(ops,
3307                             "krtld:\tsymndx\tvalue\t\t"
3308                             "symbol name\n");
3309                         lastmp = mp;
3310                 }
3311                 sp = (Sym *)(mp->symtbl +
3312                     index * mp->symhdr->sh_entsize);
3313                 _kobj_printf(ops, "krtld:\t[%3d]", index);
3314                 _kobj_printf(ops, "\t0x%lx", sp->st_value);
3315                 _kobj_printf(ops, "\t%s\n", name);
3316         }
3317 #endif
3318
3319         for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3320             ip = &mp->chains[*ip]) {
3321                 ;
3322         }
3323         *ip = index;
3324 }
3325
3326 struct modctl *
3327 kobj_boot_mod_lookup(const char *modname)
3328 {
3329         struct modctl *mctl = kobj_modules;
3330
3331         do {
3332                 if (strcmp(modname, mctl->mod_modname) == 0)
3333                         return (mctl);
3334         } while ((mctl = mctl->mod_next) != kobj_modules);
3335
3336         return (NULL);
3337 }
3338
3339 /*
3340  * Determine if the module exists.
3341  */
3342 int
3343 kobj_path_exists(char *name, int use_path)
3344 {
3345         struct _buf *file;
3346
3347         file = kobj_open_path(name, use_path);
3348         if (file == (struct _buf *)-1)
3349                 return (0);
3350         kobj_close_file(file);
3351         return (1);
3352 }
3353
3354 /*
3355  * fullname is dynamically allocated to be able to hold the
3356  * maximum size string that can be constructed from name.
3357  * path is exactly like the shell PATH variable.
3358  */
3359 struct _buf *
3360 kobj_open_path(char *name, int use_path)
3361 {
3362         char *p, *q;
3363         char *pathp;
3364         char *pathpsave;
3365         char *fullname;
3366         int maxpathlen;
3367         struct _buf *file;
3368
3369         if (!use_path)
3370                 pathp = "";             /* use name as specified */
3371         else
3372                 pathp = kobj_module_path;
3373                                         /* use configured default path */
3374
3375         pathpsave = pathp;              /* keep this for error reporting */
3376
3377         /*
3378          * Allocate enough space for the largest possible fullname.
3379          * since path is of the form <directory> : <directory> : ...
3380          * we're potentially allocating a little more than we need to
3381          * but we'll allocate the exact amount when we find the right directory.
3382          * (The + 3 below is one for NULL terminator and one for the '/'
3383          * we might have to add at the beginning of path and one for
3384          * the '/' between path and name.)
3385          */
3386         maxpathlen = strlen(pathp) + strlen(name) + 3;
3387         fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3388
3389         for (;;) {
3390                 p = fullname;
3391                 if (*pathp != '\0' && *pathp != '/')
3392                         *p++ = '/';     /* path must start with '/' */
3393                 while (*pathp && *pathp != ':' && *pathp != ' ')
3394                         *p++ = *pathp++;
3395                 if (p != fullname && p[-1] != '/')
3396                         *p++ = '/';
3397                 q = name;
3398                 while (*q)
3399                         *p++ = *q++;
3400                 *p = 0;
3401                 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3402                         kobj_free(fullname, maxpathlen);
3403                         return (file);
3404                 }
3405                 while (*pathp == ' ' || *pathp == ':')
3406                         pathp++;
3407                 if (*pathp == 0)
3408                         break;
3409
3410         }
3411         kobj_free(fullname, maxpathlen);
3412         if (_moddebug & MODDEBUG_ERRMSG) {
3413                 _kobj_printf(ops, "can't open %s,", name);
3414                 _kobj_printf(ops, " path is %s\n", pathpsave);
3415         }
3416         return ((struct _buf *)-1);
3417 }
3418
3419 intptr_t
3420 kobj_open(char *filename)
3421 {
3422         struct vnode *vp;
3423         int fd;
3424
3425         if (_modrootloaded) {
3426                 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3427                 int Errno;
3428
3429                 /*
3430                  * Hand off the open to a thread who has a
3431                  * stack size capable handling the request.
3432                  */
3433                 if (curthread != &t0) {
3434                         (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3435                             kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3436                         sema_p(&ltp->sema);
3437                         Errno = ltp->Errno;
3438                         vp = ltp->vp;
3439                 } else {
3440                         /*
3441                          * 1098067: module creds should not be those of the
3442                          * caller
3443                          */
3444                         cred_t *saved_cred = curthread->t_cred;
3445                         curthread->t_cred = kcred;
3446                         Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3447                             0, 0, rootdir, -1);
3448                         curthread->t_cred = saved_cred;
3449                 }
3450                 kobjopen_free(ltp);
3451
3452                 if (Errno) {
3453                         if (_moddebug & MODDEBUG_ERRMSG) {
3454                                 _kobj_printf(ops,
3455                                     "kobj_open: vn_open of %s fails, ",
3456                                     filename);
3457                                 _kobj_printf(ops, "Errno = %d\n", Errno);
3458                         }
3459                         return (-1);
3460                 } else {
3461                         if (_moddebug & MODDEBUG_ERRMSG) {
3462                                 _kobj_printf(ops, "kobj_open: '%s'", filename);
3463                                 _kobj_printf(ops, " vp = %p\n", vp);
3464                         }
3465                         return ((intptr_t)vp);
3466                 }
3467         } else {
3468                 fd = kobj_boot_open(filename, 0);
3469
3470                 if (_moddebug & MODDEBUG_ERRMSG) {
3471                         if (fd < 0)
3472                                 _kobj_printf(ops,
3473                                     "kobj_open: can't open %s\n", filename);
3474                         else {
3475                                 _kobj_printf(ops, "kobj_open: '%s'", filename);
3476                                 _kobj_printf(ops, " descr = 0x%x\n", fd);
3477                         }
3478                 }
3479                 return ((intptr_t)fd);
3480         }
3481 }
3482
3483 /*
3484  * Calls to kobj_open() are handled off to this routine as a separate thread.
3485  */
3486 static void
3487 kobjopen_thread(struct kobjopen_tctl *ltp)
3488 {
3489         kmutex_t        cpr_lk;
3490         callb_cpr_t     cpr_i;
3491
3492         mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3493         CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3494         ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3495             0, 0);
3496         sema_v(&ltp->sema);
3497         mutex_enter(&cpr_lk);
3498         CALLB_CPR_EXIT(&cpr_i);
3499         mutex_destroy(&cpr_lk);
3500         thread_exit();
3501 }
3502
3503 /*
3504  * allocate and initialize a kobjopen thread structure
3505  */
3506 static struct kobjopen_tctl *
3507 kobjopen_alloc(char *filename)
3508 {
3509         struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3510
3511         ASSERT(filename != NULL);
3512
3513         ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3514         bcopy(filename, ltp->name, strlen(filename) + 1);
3515         sema_init(&ltp->sema, 0, NULL, SEMA_DEFAULT, NULL);
3516         return (ltp);
3517 }
3518
3519 /*
3520  * free a kobjopen thread control structure
3521  */
3522 static void
3523 kobjopen_free(struct kobjopen_tctl *ltp)
3524 {
3525         sema_destroy(&ltp->sema);
3526         kmem_free(ltp->name, strlen(ltp->name) + 1);
3527         kmem_free(ltp, sizeof (*ltp));
3528 }
3529
3530 int
3531 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3532 {
3533         int stat;
3534         ssize_t resid;
3535
3536         if (_modrootloaded) {
3537                 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3538                     (offset_t)offset, UIO_SYSSPACE, 0, 0, CRED(),
3539                     &resid)) != 0) {
3540                         _kobj_printf(ops,
3541                             "vn_rdwr failed with error 0x%x\n", stat);
3542                         return (-1);
3543                 }
3544                 return (size - resid);
3545         } else {
3546                 int count = 0;
3547
3548                 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3549                         _kobj_printf(ops,
3550                             "kobj_read: seek 0x%x failed\n", offset);
3551                         return (-1);
3552                 }
3553
3554                 count = kobj_boot_read((int)descr, buf, size);
3555                 if (count < size) {
3556                         if (_moddebug & MODDEBUG_ERRMSG) {
3557                                 _kobj_printf(ops,
3558                                     "kobj_read: req %d bytes, ", size);
3559                                 _kobj_printf(ops, "got %d\n", count);
3560                         }
3561                 }
3562                 return (count);
3563         }
3564 }
3565
3566 void
3567 kobj_close(intptr_t descr)
3568 {
3569         if (_moddebug & MODDEBUG_ERRMSG)
3570                 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3571
3572         if (_modrootloaded) {
3573                 struct vnode *vp = (struct vnode *)descr;
3574                 (void) fop_close(vp, FREAD, 1, 0, CRED(), NULL);
3575                 VN_RELE(vp);
3576         } else
3577                 (void) kobj_boot_close((int)descr);
3578 }
3579
3580 int
3581 kobj_fstat(intptr_t descr, struct bootstat *buf)
3582 {
3583         if (buf == NULL)
3584                 return (-1);
3585
3586         if (_modrootloaded) {
3587                 vattr_t vattr;
3588                 struct vnode *vp = (struct vnode *)descr;
3589                 if (fop_getattr(vp, &vattr, 0, kcred, NULL) != 0)
3590                         return (-1);
3591
3592                 /*
3593                  * The vattr and bootstat structures are similar, but not
3594                  * identical.  We do our best to fill in the bootstat structure
3595                  * from the contents of vattr (transfering only the ones that
3596                  * are obvious.
3597                  */
3598
3599                 buf->st_mode = (uint32_t)vattr.va_mode;
3600                 buf->st_nlink = (uint32_t)vattr.va_nlink;
3601                 buf->st_uid = (int32_t)vattr.va_uid;
3602                 buf->st_gid = (int32_t)vattr.va_gid;
3603                 buf->st_rdev = (uint64_t)vattr.va_rdev;
3604                 buf->st_size = (uint64_t)vattr.va_size;
3605                 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3606                 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3607                 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3608                 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3609                 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3610                 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3611                 buf->st_blksize = (int32_t)vattr.va_blksize;
3612                 buf->st_blocks = (int64_t)vattr.va_nblocks;
3613
3614                 return (0);
3615         }
3616
3617         return (kobj_boot_fstat((int)descr, buf));
3618 }
3619
3620
3621 struct _buf *
3622 kobj_open_file(char *name)
3623 {
3624         struct _buf *file;
3625         struct compinfo cbuf;
3626         intptr_t fd;
3627
3628         if ((fd = kobj_open(name)) == -1) {
3629                 return ((struct _buf *)-1);
3630         }
3631
3632         file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3633         file->_fd = fd;
3634         file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3635         file->_cnt = file->_size = file->_off = 0;
3636         file->_ln = 1;
3637         file->_ptr = file->_base;
3638         (void) strcpy(file->_name, name);
3639
3640         /*
3641          * Before root is mounted, we must check
3642          * for a compressed file and do our own
3643          * buffering.
3644          */
3645         if (_modrootloaded) {
3646                 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3647                 file->_bsize = MAXBSIZE;
3648
3649                 /* Check if the file is compressed */
3650                 file->_iscmp = kobj_is_compressed(fd);
3651         } else {
3652                 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3653                         kobj_close_file(file);
3654                         return ((struct _buf *)-1);
3655                 }
3656                 file->_iscmp = cbuf.iscmp;
3657                 if (file->_iscmp) {
3658                         if (kobj_comp_setup(file, &cbuf) != 0) {
3659                                 kobj_close_file(file);
3660                                 return ((struct _buf *)-1);
3661                         }
3662                 } else {
3663                         file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3664                         file->_bsize = cbuf.blksize;
3665                 }
3666         }
3667         return (file);
3668 }
3669
3670 static int
3671 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3672 {
3673         struct comphdr *hdr;
3674
3675         /*
3676          * read the compressed image into memory,
3677          * so we can deompress from there
3678          */
3679         file->_dsize = cip->fsize;
3680         file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3681         if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3682                 kobj_free(file->_dbuf, cip->fsize);
3683                 return (-1);
3684         }
3685
3686         hdr = kobj_comphdr(file);
3687         if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3688             hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3689             !ISP2(hdr->ch_blksize)) {
3690                 kobj_free(file->_dbuf, cip->fsize);
3691                 return (-1);
3692         }
3693         file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3694         file->_bsize = hdr->ch_blksize;
3695         return (0);
3696 }
3697
3698 void
3699 kobj_close_file(struct _buf *file)
3700 {
3701         kobj_close(file->_fd);
3702         if (file->_base != NULL)
3703                 kobj_free(file->_base, file->_bsize);
3704         if (file->_dbuf != NULL)
3705                 kobj_free(file->_dbuf, file->_dsize);
3706         kobj_free(file->_name, strlen(file->_name)+1);
3707         kobj_free(file, sizeof (struct _buf));
3708 }
3709
3710 int
3711 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3712 {
3713         int b_size, c_size;
3714         int b_off;      /* Offset into buffer for start of bcopy */
3715         int count = 0;
3716         int page_addr;
3717
3718         if (_moddebug & MODDEBUG_ERRMSG) {
3719                 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3720                 _kobj_printf(ops, " offset=%x at", off);
3721                 _kobj_printf(ops, " buf=%x\n", buf);
3722         }
3723
3724         /*
3725          * Handle compressed (gzip for now) file here. First get the
3726          * compressed size, then read the image into memory and finally
3727          * call zlib to decompress the image at the supplied memory buffer.
3728          */
3729         if (file->_iscmp == CH_MAGIC_GZIP) {
3730                 ulong_t dlen;
3731                 vattr_t vattr;
3732                 struct vnode *vp = (struct vnode *)file->_fd;
3733                 ssize_t resid;
3734                 int err = 0;
3735
3736                 if (fop_getattr(vp, &vattr, 0, kcred, NULL) != 0)
3737                         return (-1);
3738
3739                 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3740                 file->_dsize = vattr.va_size;
3741
3742                 /* Read the compressed file into memory */
3743                 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3744                     (offset_t)(0), UIO_SYSSPACE, 0, 0, CRED(),
3745                     &resid)) != 0) {
3746
3747                         _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3748                             "error code 0x%x\n", err);
3749                         return (-1);
3750                 }
3751
3752                 dlen = size;
3753
3754                 /* Decompress the image at the supplied memory buffer */
3755                 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3756                     vattr.va_size)) != Z_OK) {
3757                         _kobj_printf(ops, "kobj_read_file: z_uncompress "
3758                             "failed, error code : 0x%x\n", err);
3759                         return (-1);
3760                 }
3761
3762                 if (dlen != size) {
3763                         _kobj_printf(ops, "kobj_read_file: z_uncompress "
3764                             "failed to uncompress (size returned 0x%x , "
3765                             "expected size: 0x%x)\n", dlen, size);
3766                         return (-1);
3767                 }
3768
3769                 return (0);
3770         }
3771
3772         while (size) {
3773                 page_addr = F_PAGE(file, off);
3774                 b_size = file->_size;
3775                 /*
3776                  * If we have the filesystem page the caller's referring to
3777                  * and we have something in the buffer,
3778                  * satisfy as much of the request from the buffer as we can.
3779                  */
3780                 if (page_addr == file->_off && b_size > 0) {
3781                         b_off = B_OFFSET(file, off);
3782                         c_size = b_size - b_off;
3783                         /*
3784                          * If there's nothing to copy, we're at EOF.
3785                          */
3786                         if (c_size <= 0)
3787                                 break;
3788                         if (c_size > size)
3789                                 c_size = size;
3790                         if (buf) {
3791                                 if (_moddebug & MODDEBUG_ERRMSG)
3792                                         _kobj_printf(ops, "copying %x bytes\n",
3793                                             c_size);
3794                                 bcopy(file->_base+b_off, buf, c_size);
3795                                 size -= c_size;
3796                                 off += c_size;
3797                                 buf += c_size;
3798                                 count += c_size;
3799                         } else {
3800                                 _kobj_printf(ops, "kobj_read: system error");
3801                                 count = -1;
3802                                 break;
3803                         }
3804                 } else {
3805                         /*
3806                          * If the caller's offset is page aligned and
3807                          * the caller want's at least a filesystem page and
3808                          * the caller provided a buffer,
3809                          * read directly into the caller's buffer.
3810                          */
3811                         if (page_addr == off &&
3812                             (c_size = F_BLKS(file, size)) && buf) {
3813                                 c_size = kobj_read_blks(file, buf, c_size,
3814                                     page_addr);
3815                                 if (c_size < 0) {
3816                                         count = -1;
3817                                         break;
3818                                 }
3819                                 count += c_size;
3820                                 if (c_size != F_BLKS(file, size))
3821                                         break;
3822                                 size -= c_size;
3823                                 off += c_size;
3824                                 buf += c_size;
3825                         /*
3826                          * Otherwise, read into our buffer and copy next time
3827                          * around the loop.
3828                          */
3829                         } else {
3830                                 file->_off = page_addr;
3831                                 c_size = kobj_read_blks(file, file->_base,
3832                                     file->_bsize, page_addr);
3833                                 file->_ptr = file->_base;
3834                                 file->_cnt = c_size;
3835                                 file->_size = c_size;
3836                                 /*
3837                                  * If a _filbuf call or nothing read, break.
3838                                  */
3839                                 if (buf == NULL || c_size <= 0) {
3840                                         count = c_size;
3841                                         break;
3842                                 }
3843                         }
3844                         if (_moddebug & MODDEBUG_ERRMSG)
3845                                 _kobj_printf(ops, "read %x bytes\n", c_size);
3846                 }
3847         }
3848         if (_moddebug & MODDEBUG_ERRMSG)
3849                 _kobj_printf(ops, "count = %x\n", count);
3850
3851         return (count);
3852 }
3853
3854 static int
3855 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3856 {
3857         int ret;
3858
3859         ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3860         if (file->_iscmp) {
3861                 uint_t blks;
3862                 int nret;
3863
3864                 ret = 0;
3865                 for (blks = size / file->_bsize; blks != 0; blks--) {
3866                         nret = kobj_uncomp_blk(file, buf, off);
3867                         if (nret == -1)
3868                                 return (-1);
3869                         buf += nret;
3870                         off += nret;
3871                         ret += nret;
3872                         if (nret < file->_bsize)
3873                                 break;
3874                 }
3875         } else
3876                 ret = kobj_read(file->_fd, buf, size, off);
3877         return (ret);
3878 }
3879
3880 static int
3881 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
3882 {
3883         struct comphdr *hdr = kobj_comphdr(file);
3884         ulong_t dlen, slen;
3885         caddr_t src;
3886         int i;
3887
3888         dlen = file->_bsize;
3889         i = off / file->_bsize;
3890         src = file->_dbuf + hdr->ch_blkmap[i];
3891         if (i == hdr->ch_fsize / file->_bsize)
3892                 slen = file->_dsize - hdr->ch_blkmap[i];
3893         else
3894                 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
3895         if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
3896                 return (-1);
3897         return (dlen);
3898 }
3899
3900 int
3901 kobj_filbuf(struct _buf *f)
3902 {
3903         if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
3904                 return (kobj_getc(f));
3905         return (-1);
3906 }
3907
3908 void
3909 kobj_free(void *address, size_t size)
3910 {
3911         if (standalone)
3912                 return;
3913
3914         kmem_free(address, size);
3915         kobj_stat.nfree_calls++;
3916         kobj_stat.nfree += size;
3917 }
3918
3919 void *
3920 kobj_zalloc(size_t size, int flag)
3921 {
3922         void *v;
3923
3924         if ((v = kobj_alloc(size, flag)) != 0) {
3925                 bzero(v, size);
3926         }
3927
3928         return (v);
3929 }
3930
3931 void *
3932 kobj_alloc(size_t size, int flag)
3933 {
3934         /*
3935          * If we are running standalone in the
3936          * linker, we ask boot for memory.
3937          * Either it's temporary memory that we lose
3938          * once boot is mapped out or we allocate it
3939          * permanently using the dynamic data segment.
3940          */
3941         if (standalone) {
3942 #if defined(_OBP)
3943                 if (flag & (KM_TMP | KM_SCRATCH))
3944                         return (bop_temp_alloc(size, MINALIGN));
3945 #else
3946                 if (flag & (KM_TMP | KM_SCRATCH))
3947                         return (BOP_ALLOC(ops, 0, size, MINALIGN));
3948 #endif
3949                 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
3950         }
3951
3952         kobj_stat.nalloc_calls++;
3953         kobj_stat.nalloc += size;
3954
3955         return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
3956 }
3957
3958 /*
3959  * Allow the "mod" system to sync up with the work
3960  * already done by kobj during the initial loading
3961  * of the kernel.  This also gives us a chance
3962  * to reallocate memory that belongs to boot.
3963  */
3964 void
3965 kobj_sync(void)
3966 {
3967         struct modctl_list *lp, **lpp;
3968
3969         /*
3970          * The module path can be set in /etc/system via 'moddir' commands
3971          */
3972         if (default_path != NULL)
3973                 kobj_module_path = default_path;
3974         else
3975                 default_path = kobj_module_path;
3976
3977         ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
3978             segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
3979
3980         ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
3981             segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
3982
3983         /*
3984          * Move symbol tables from boot memory to ksyms_arena.
3985          */
3986         for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
3987                 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
3988                         kobj_export_module(mod(lp));
3989         }
3990 }
3991
3992 caddr_t
3993 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
3994 {
3995         uintptr_t va, pva;
3996         size_t alloc_pgsz = kobj_mmu_pagesize;
3997         size_t alloc_align = BO_NO_ALIGN;
3998         size_t alloc_size;
3999
4000         /*
4001          * If we are using "large" mappings for the kernel,
4002          * request aligned memory from boot using the
4003          * "large" pagesize.
4004          */
4005         if (lg_pagesize) {
4006                 alloc_align = lg_pagesize;
4007                 alloc_pgsz = lg_pagesize;
4008         }
4009
4010
4011         va = ALIGN((uintptr_t)*spp, align);
4012         pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4013         /*
4014          * Need more pages?
4015          */
4016         if (va + size > pva) {
4017                 uintptr_t npva;
4018
4019                 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4020                 /*
4021                  * Check for overlapping segments.
4022                  */
4023                 if (limit && limit <= *spp + alloc_size) {
4024                         return ((caddr_t)0);
4025                 }
4026
4027                 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4028                     alloc_size, alloc_align);
4029
4030                 if (npva == (uintptr_t)NULL) {
4031                         _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4032                             alloc_size);
4033                         _kobj_printf(ops, " aligned %lx", alloc_align);
4034                         _kobj_printf(ops, " at 0x%lx\n", pva);
4035                         return (NULL);
4036                 }
4037         }
4038         *spp = (caddr_t)(va + size);
4039
4040         return ((caddr_t)va);
4041 }
4042
4043 /*
4044  * Calculate the number of output hash buckets.
4045  * We use the next prime larger than n / 4,
4046  * so the average hash chain is about 4 entries.
4047  * More buckets would just be a waste of memory.
4048  */
4049 uint_t
4050 kobj_gethashsize(uint_t n)
4051 {
4052         int f;
4053         int hsize = MAX(n / 4, 2);
4054
4055         for (f = 2; f * f <= hsize; f++)
4056                 if (hsize % f == 0)
4057                         hsize += f = 1;
4058
4059         return (hsize);
4060 }
4061
4062 /*
4063  * Get the file size.
4064  *
4065  * Before root is mounted, files are compressed in the boot_archive ramdisk
4066  * (in the memory). kobj_fstat would return the compressed file size.
4067  * In order to get the uncompressed file size, read the file to the end and
4068  * count its size.
4069  */
4070 int
4071 kobj_get_filesize(struct _buf *file, uint64_t *size)
4072 {
4073         int err = 0;
4074         ssize_t resid;
4075         uint32_t buf;
4076
4077         if (_modrootloaded) {
4078                 struct bootstat bst;
4079
4080                 if (kobj_fstat(file->_fd, &bst) != 0)
4081                         return (EIO);
4082                 *size = bst.st_size;
4083
4084                 if (file->_iscmp == CH_MAGIC_GZIP) {
4085                         /*
4086                          * Read the last 4 bytes of the compressed (gzip)
4087                          * image to get the size of its uncompressed
4088                          * version.
4089                          */
4090                         if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4091                             (char *)(&buf), 4, (offset_t)(*size - 4),
4092                             UIO_SYSSPACE, 0, 0, CRED(), &resid))
4093                             != 0) {
4094                                 _kobj_printf(ops, "kobj_get_filesize: "
4095                                     "vn_rdwr() failed with error 0x%x\n", err);
4096                                 return (-1);
4097                         }
4098
4099                         *size =  (uint64_t)buf;
4100                 }
4101         } else {
4102
4103 #if defined(_OBP)
4104                 struct bootstat bsb;
4105
4106                 if (file->_iscmp) {
4107                         struct comphdr *hdr = kobj_comphdr(file);
4108
4109                         *size = hdr->ch_fsize;
4110                 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4111                         return (EIO);
4112                 else
4113                         *size = bsb.st_size;
4114 #else
4115                 char *buf;
4116                 int count;
4117                 uint64_t offset = 0;
4118
4119                 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4120                 do {
4121                         count = kobj_read_file(file, buf, MAXBSIZE, offset);
4122                         if (count < 0) {
4123                                 kmem_free(buf, MAXBSIZE);
4124                                 return (EIO);
4125                         }
4126                         offset += count;
4127                 } while (count == MAXBSIZE);
4128                 kmem_free(buf, MAXBSIZE);
4129
4130                 *size = offset;
4131 #endif
4132         }
4133
4134         return (0);
4135 }
4136
4137 static char *
4138 basename(char *s)
4139 {
4140         char *p, *q;
4141
4142         q = NULL;
4143         p = s;
4144         do {
4145                 if (*p == '/')
4146                         q = p;
4147         } while (*p++);
4148         return (q ? q + 1 : s);
4149 }
4150
4151 void
4152 kobj_stat_get(kobj_stat_t *kp)
4153 {
4154         *kp = kobj_stat;
4155 }
4156
4157 int
4158 kobj_getpagesize()
4159 {
4160         return (lg_pagesize);
4161 }
4162
4163 void
4164 kobj_textwin_alloc(struct module *mp)
4165 {
4166         ASSERT(MUTEX_HELD(&mod_lock));
4167
4168         if (mp->textwin != NULL)
4169                 return;
4170
4171         /*
4172          * If the text is not contained in the heap, then it is not contained
4173          * by a writable mapping.  (Specifically, it's on the nucleus page.)
4174          * We allocate a read/write mapping for this module's text to allow
4175          * the text to be patched without calling hot_patch_kernel_text()
4176          * (which is quite slow).
4177          */
4178         if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4179                 uintptr_t text = (uintptr_t)mp->text;
4180                 uintptr_t size = (uintptr_t)mp->text_size;
4181                 uintptr_t i;
4182                 caddr_t va;
4183                 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4184                     (text & PAGEMASK);
4185
4186                 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4187
4188                 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4189                         hat_devload(kas.a_hat, va, PAGESIZE,
4190                             hat_getpfnum(kas.a_hat, (caddr_t)i),
4191                             PROT_READ | PROT_WRITE,
4192                             HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4193                         va += PAGESIZE;
4194                 }
4195
4196                 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4197         } else {
4198                 mp->textwin = mp->text;
4199         }
4200 }
4201
4202 void
4203 kobj_textwin_free(struct module *mp)
4204 {
4205         uintptr_t text = (uintptr_t)mp->text;
4206         uintptr_t tsize = (uintptr_t)mp->text_size;
4207         size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4208             (text & PAGEMASK));
4209
4210         mp->textwin = NULL;
4211
4212         if (mp->textwin_base == NULL)
4213                 return;
4214
4215         hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4216         vmem_free(heap_arena, mp->textwin_base, size);
4217         mp->textwin_base = NULL;
4218 }
4219
4220 static char *
4221 find_libmacro(char *name)
4222 {
4223         int lmi;
4224
4225         for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4226                 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4227                         return (libmacros[lmi].lmi_list);
4228         }
4229         return (NULL);
4230 }
4231
4232 /*
4233  * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4234  * returns path if successful, else NULL
4235  * Support multiple $MACROs expansion and the first valid path will be returned
4236  * Caller's responsibility to provide enough space in path to expand
4237  */
4238 char *
4239 expand_libmacro(char *tail, char *path, char *pathend)
4240 {
4241         char c, *p, *p1, *p2, *path2, *endp;
4242         int diff, lmi, macrolen, valid_macro, more_macro;
4243         struct _buf *file;
4244
4245         /*
4246          * check for $MACROS between nulls or slashes
4247          */
4248         p = strchr(tail, '$');
4249         if (p == NULL)
4250                 return (NULL);
4251         for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4252                 macrolen = libmacros[lmi].lmi_macrolen;
4253                 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4254                         break;
4255         }
4256
4257         valid_macro = 0;
4258         if (lmi < NLIBMACROS) {
4259                 /*
4260                  * The following checks are used to restrict expansion of
4261                  * macros to those that form a full directory/file name
4262                  * and to keep the behavior same as before.  If this
4263                  * restriction is removed or no longer valid in the future,
4264                  * the checks below can be deleted.
4265                  */
4266                 if ((p == tail) || (*(p - 1) == '/')) {
4267                         c = *(p + macrolen + 1);
4268                         if (c == '/' || c == '\0')
4269                                 valid_macro = 1;
4270                 }
4271         }
4272
4273         if (!valid_macro) {
4274                 p2 = strchr(p, '/');
4275                 /*
4276                  * if no more macro to expand, then just copy whatever left
4277                  * and check whether it exists
4278                  */
4279                 if (p2 == NULL || strchr(p2, '$') == NULL) {
4280                         (void) strcpy(pathend, tail);
4281                         if ((file = kobj_open_path(path, 1)) !=
4282                             (struct _buf *)-1) {
4283                                 kobj_close_file(file);
4284                                 return (path);
4285                         } else
4286                                 return (NULL);
4287                 } else {
4288                         /*
4289                          * copy all chars before '/' and call expand_libmacro()
4290                          * again
4291                          */
4292                         diff = p2 - tail;
4293                         bcopy(tail, pathend, diff);
4294                         pathend += diff;
4295                         *(pathend) = '\0';
4296                         return (expand_libmacro(p2, path, pathend));
4297                 }
4298         }
4299
4300         more_macro = 0;
4301         if (c != '\0') {
4302                 endp = p + macrolen + 1;
4303                 if (strchr(endp, '$') != NULL)
4304                         more_macro = 1;
4305         } else
4306                 endp = NULL;
4307
4308         /*
4309          * copy lmi_list and split it into components.
4310          * then put the part of tail before $MACRO into path
4311          * at pathend
4312          */
4313         diff = p - tail;
4314         if (diff > 0)
4315                 bcopy(tail, pathend, diff);
4316         path2 = pathend + diff;
4317         p1 = libmacros[lmi].lmi_list;
4318         while (p1 && (*p1 != '\0')) {
4319                 p2 = strchr(p1, ':');
4320                 if (p2) {
4321                         diff = p2 - p1;
4322                         bcopy(p1, path2, diff);
4323                         *(path2 + diff) = '\0';
4324                 } else {
4325                         diff = strlen(p1);
4326                         bcopy(p1, path2, diff + 1);
4327                 }
4328                 /* copy endp only if there isn't any more macro to expand */
4329                 if (!more_macro && (endp != NULL))
4330                         (void) strcat(path2, endp);
4331                 file = kobj_open_path(path, 1);
4332                 if (file != (struct _buf *)-1) {
4333                         kobj_close_file(file);
4334                         /*
4335                          * if more macros to expand then call expand_libmacro(),
4336                          * else return path which has the whole path
4337                          */
4338                         if (!more_macro || (expand_libmacro(endp, path,
4339                             path2 + diff) != NULL)) {
4340                                 return (path);
4341                         }
4342                 }
4343                 if (p2)
4344                         p1 = ++p2;
4345                 else
4346                         return (NULL);
4347         }
4348         return (NULL);
4349 }
4350
4351 static void
4352 tnf_add_notifyunload(kobj_notify_f *fp)
4353 {
4354         kobj_notify_list_t *entry;
4355
4356         entry = kobj_alloc(sizeof (kobj_notify_list_t), KM_WAIT);
4357         entry->kn_type = KOBJ_NOTIFY_MODUNLOADING;
4358         entry->kn_func = fp;
4359         (void) kobj_notify_add(entry);
4360 }
4361
4362 /* ARGSUSED */
4363 static void
4364 tnf_unsplice_probes(uint_t what, struct modctl *mod)
4365 {
4366         tnf_probe_control_t **p;
4367         tnf_tag_data_t **q;
4368         struct module *mp = mod->mod_mp;
4369
4370         if (!(mp->flags & KOBJ_TNF_PROBE))
4371                 return;
4372
4373         for (p = &__tnf_probe_list_head; *p; )
4374                 if (kobj_addrcheck(mp, (char *)*p) == 0)
4375                         *p = (*p)->next;
4376                 else
4377                         p = &(*p)->next;
4378
4379         for (q = &__tnf_tag_list_head; *q; )
4380                 if (kobj_addrcheck(mp, (char *)*q) == 0)
4381                         *q = (tnf_tag_data_t *)(*q)->tag_version;
4382                 else
4383                         q = (tnf_tag_data_t **)&(*q)->tag_version;
4384
4385         tnf_changed_probe_list = 1;
4386 }
4387
4388 int
4389 tnf_splice_probes(int boot_load, tnf_probe_control_t *plist,
4390     tnf_tag_data_t *tlist)
4391 {
4392         int result = 0;
4393         static int add_notify = 1;
4394
4395         if (plist) {
4396                 tnf_probe_control_t *pl;
4397
4398                 for (pl = plist; pl->next; )
4399                         pl = pl->next;
4400
4401                 if (!boot_load)
4402                         mutex_enter(&mod_lock);
4403                 tnf_changed_probe_list = 1;
4404                 pl->next = __tnf_probe_list_head;
4405                 __tnf_probe_list_head = plist;
4406                 if (!boot_load)
4407                         mutex_exit(&mod_lock);
4408                 result = 1;
4409         }
4410
4411         if (tlist) {
4412                 tnf_tag_data_t *tl;
4413
4414                 for (tl = tlist; tl->tag_version; )
4415                         tl = (tnf_tag_data_t *)tl->tag_version;
4416
4417                 if (!boot_load)
4418                         mutex_enter(&mod_lock);
4419                 tl->tag_version = (tnf_tag_version_t *)__tnf_tag_list_head;
4420                 __tnf_tag_list_head = tlist;
4421                 if (!boot_load)
4422                         mutex_exit(&mod_lock);
4423                 result = 1;
4424         }
4425         if (!boot_load && result && add_notify) {
4426                 tnf_add_notifyunload(tnf_unsplice_probes);
4427                 add_notify = 0;
4428         }
4429         return (result);
4430 }
4431
4432 char *kobj_file_buf;
4433 int kobj_file_bufsize;
4434
4435 /*
4436  * This code is for the purpose of manually recording which files
4437  * needs to go into the boot archive on any given system.
4438  *
4439  * To enable the code, set kobj_file_bufsize in /etc/system
4440  * and reboot the system, then use mdb to look at kobj_file_buf.
4441  */
4442 static void
4443 kobj_record_file(char *filename)
4444 {
4445         static char *buf;
4446         static int size = 0;
4447         int n;
4448
4449         if (kobj_file_bufsize == 0)     /* don't bother */
4450                 return;
4451
4452         if (kobj_file_buf == NULL) {    /* allocate buffer */
4453                 size = kobj_file_bufsize;
4454                 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4455         }
4456
4457         n = snprintf(buf, size, "%s\n", filename);
4458         if (n > size)
4459                 n = size;
4460         size -= n;
4461         buf += n;
4462 }
4463
4464 static int
4465 kobj_boot_fstat(int fd, struct bootstat *stp)
4466 {
4467 #if defined(_OBP)
4468         if (!standalone && _ioquiesced)
4469                 return (-1);
4470         return (BOP_FSTAT(ops, fd, stp));
4471 #else
4472         return (BRD_FSTAT(bfs_ops, fd, stp));
4473 #endif
4474 }
4475
4476 static int
4477 kobj_boot_open(char *filename, int flags)
4478 {
4479 #if defined(_OBP)
4480
4481         /*
4482          * If io via bootops is quiesced, it means boot is no longer
4483          * available to us.  We make it look as if we can't open the
4484          * named file - which is reasonably accurate.
4485          */
4486         if (!standalone && _ioquiesced)
4487                 return (-1);
4488
4489         kobj_record_file(filename);
4490         return (BOP_OPEN(filename, flags));
4491 #else /* x86 */
4492         kobj_record_file(filename);
4493         return (BRD_OPEN(bfs_ops, filename, flags));
4494 #endif
4495 }
4496
4497 static int
4498 kobj_boot_close(int fd)
4499 {
4500 #if defined(_OBP)
4501         if (!standalone && _ioquiesced)
4502                 return (-1);
4503
4504         return (BOP_CLOSE(fd));
4505 #else /* x86 */
4506         return (BRD_CLOSE(bfs_ops, fd));
4507 #endif
4508 }
4509
4510 /*ARGSUSED*/
4511 static int
4512 kobj_boot_seek(int fd, off_t hi, off_t lo)
4513 {
4514 #if defined(_OBP)
4515         return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4516 #else
4517         return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4518 #endif
4519 }
4520
4521 static int
4522 kobj_boot_read(int fd, caddr_t buf, size_t size)
4523 {
4524 #if defined(_OBP)
4525         return (BOP_READ(fd, buf, size));
4526 #else
4527         return (BRD_READ(bfs_ops, fd, buf, size));
4528 #endif
4529 }
4530
4531 static int
4532 kobj_boot_compinfo(int fd, struct compinfo *cb)
4533 {
4534         return (boot_compinfo(fd, cb));
4535 }
4536
4537 /*
4538  * Check if the file is compressed (for now we handle only gzip).
4539  * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4540  */
4541 static int
4542 kobj_is_compressed(intptr_t fd)
4543 {
4544         struct vnode *vp = (struct vnode *)fd;
4545         ssize_t resid;
4546         uint16_t magic_buf;
4547         int err = 0;
4548
4549         if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4550             sizeof (magic_buf), (offset_t)(0),
4551             UIO_SYSSPACE, 0, 0, CRED(), &resid)) != 0) {
4552
4553                 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4554                     "error code 0x%x\n", err);
4555                 return (0);
4556         }
4557
4558         if (magic_buf == CH_MAGIC_GZIP)
4559                 return (CH_MAGIC_GZIP);
4560
4561         return (0);
4562 }