1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2011 Bayard G. Bell <buffer.g.overflow@gmail.com>.
27 * All rights reserved. Use is subject to license terms.
28 * Copyright 2020 Joyent, Inc.
29 * Copyright 2025 MNX Cloud, Inc.
30 */
31
32 /*
33 * Kernel's linker/loader
34 */
35
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/sysmacros.h>
39 #include <sys/systm.h>
40 #include <sys/user.h>
41 #include <sys/kmem.h>
42 #include <sys/reboot.h>
43 #include <sys/bootconf.h>
44 #include <sys/debug.h>
45 #include <sys/uio.h>
46 #include <sys/file.h>
47 #include <sys/vnode.h>
48 #include <sys/user.h>
49 #include <sys/mman.h>
50 #include <vm/as.h>
51 #include <vm/seg_kp.h>
52 #include <vm/seg_kmem.h>
53 #include <sys/elf.h>
54 #include <sys/elf_notes.h>
55 #include <sys/vmsystm.h>
56 #include <sys/kdi.h>
57 #include <sys/atomic.h>
58 #include <sys/kmdb.h>
59
60 #include <sys/link.h>
61 #include <sys/kobj.h>
62 #include <sys/ksyms.h>
63 #include <sys/disp.h>
64 #include <sys/modctl.h>
65 #include <sys/varargs.h>
66 #include <sys/kstat.h>
67 #include <sys/kobj_impl.h>
68 #include <sys/fs/decomp.h>
69 #include <sys/callb.h>
70 #include <sys/cmn_err.h>
71 #include <sys/zmod.h>
72
73 #include <krtld/reloc.h>
74 #include <krtld/kobj_kdi.h>
75 #include <sys/sha1.h>
76 #include <sys/crypto/elfsign.h>
77
78 #if !defined(_OBP)
79 #include <sys/bootvfs.h>
80 #endif
81
82 /*
83 * do_symbols() error codes
84 */
85 #define DOSYM_UNDEF -1 /* undefined symbol */
86 #define DOSYM_UNSAFE -2 /* MT-unsafe driver symbol */
87
88 #if !defined(_OBP)
89 static void synthetic_bootaux(char *, val_t *);
90 #endif
91
92 static struct module *load_exec(val_t *, char *);
93 static void load_linker(val_t *);
94 static struct modctl *add_primary(const char *filename, int);
95 static int bind_primary(val_t *, int);
96 static int load_primary(struct module *, int);
97 static int load_kmdb(val_t *);
98 static int get_progbits(struct module *, struct _buf *);
99 static int get_syms(struct module *, struct _buf *);
100 static int get_ctf(struct module *, struct _buf *);
101 static void get_signature(struct module *, struct _buf *);
102 static int do_common(struct module *);
103 static void add_dependent(struct module *, struct module *);
104 static int do_dependents(struct modctl *, char *, size_t);
105 static int do_symbols(struct module *, Elf64_Addr);
106 static void module_assign(struct modctl *, struct module *);
107 static void free_module_data(struct module *);
108 static char *depends_on(struct module *);
109 static char *getmodpath(const char *);
110 static char *basename(char *);
111 static void attr_val(val_t *);
112 static char *find_libmacro(char *);
113 static char *expand_libmacro(char *, char *, char *);
114 static int read_bootflags(void);
115 static int kobj_comp_setup(struct _buf *, struct compinfo *);
116 static int kobj_uncomp_blk(struct _buf *, caddr_t, uint_t);
117 static int kobj_read_blks(struct _buf *, caddr_t, uint_t, uint_t);
118 static int kobj_boot_open(char *, int);
119 static int kobj_boot_close(int);
120 static int kobj_boot_seek(int, off_t, off_t);
121 static int kobj_boot_read(int, caddr_t, size_t);
122 static int kobj_boot_fstat(int, struct bootstat *);
123 static int kobj_boot_compinfo(int, struct compinfo *);
124
125 static Sym *lookup_one(struct module *, const char *);
126 static void sym_insert(struct module *, char *, symid_t);
127 static Sym *sym_lookup(struct module *, Sym *);
128
129 static struct kobjopen_tctl *kobjopen_alloc(char *filename);
130 static void kobjopen_free(struct kobjopen_tctl *ltp);
131 static void kobjopen_thread(struct kobjopen_tctl *ltp);
132 static int kobj_is_compressed(intptr_t);
133
134 extern int kcopy(const void *, void *, size_t);
135 extern int elf_mach_ok(Ehdr *);
136 extern int alloc_gottable(struct module *, caddr_t *, caddr_t *);
137
138 #if !defined(_OBP)
139 extern int kobj_boot_mountroot(void);
140 #endif
141
142 extern int modrootloaded;
143 extern int swaploaded;
144 extern int bop_io_quiesced;
145 extern int last_module_id;
146
147 extern char stubs_base[];
148 extern char stubs_end[];
149
150 #ifdef KOBJ_DEBUG
151 /*
152 * Values that can be or'd in to kobj_debug and their effects:
153 *
154 * D_DEBUG - misc. debugging information.
155 * D_SYMBOLS - list symbols and their values as they are entered
156 * into the hash table
157 * D_RELOCATIONS - display relocation processing information
158 * D_LOADING - display information about each module as it
159 * is loaded.
160 */
161 int kobj_debug = 0;
162
163 #define KOBJ_MARK(s) if (kobj_debug & D_DEBUG) \
164 (_kobj_printf(ops, "%d", __LINE__), _kobj_printf(ops, ": %s\n", s))
165 #else
166 #define KOBJ_MARK(s) /* discard */
167 #endif
168
169 #define MODPATH_PROPNAME "module-path"
170
171 #ifdef MODDIR_SUFFIX
172 static char slash_moddir_suffix_slash[] = MODDIR_SUFFIX "/";
173 #else
174 #define slash_moddir_suffix_slash ""
175 #endif
176
177 #define _moddebug get_weakish_int(&moddebug)
178 #define _modrootloaded get_weakish_int(&modrootloaded)
179 #define _swaploaded get_weakish_int(&swaploaded)
180 #define _ioquiesced get_weakish_int(&bop_io_quiesced)
181
182 #define mod(X) (struct module *)((X)->modl_modp->mod_mp)
183
184 void *romp; /* rom vector (opaque to us) */
185 struct bootops *ops; /* bootops vector */
186 void *dbvec; /* debug vector */
187
188 /*
189 * kobjopen thread control structure
190 */
191 struct kobjopen_tctl {
192 ksema_t sema;
193 char *name; /* name of file */
194 struct vnode *vp; /* vnode return from vn_open() */
195 int Errno; /* error return from vnopen */
196 };
197
198 /*
199 * Structure for defining dynamically expandable library macros
200 */
201
202 struct lib_macro_info {
203 char *lmi_list; /* ptr to list of possible choices */
204 char *lmi_macroname; /* pointer to macro name */
205 ushort_t lmi_ba_index; /* index into bootaux vector */
206 ushort_t lmi_macrolen; /* macro length */
207 } libmacros[] = {
208 { NULL, "CPU", BA_CPU, 0 },
209 { NULL, "MMU", BA_MMU, 0 }
210 };
211
212 #define NLIBMACROS sizeof (libmacros) / sizeof (struct lib_macro_info)
213
214 char *boot_cpu_compatible_list; /* make $CPU available */
215
216 char *kobj_module_path; /* module search path */
217 vmem_t *text_arena; /* module text arena */
218 static vmem_t *data_arena; /* module data & bss arena */
219 static vmem_t *ctf_arena; /* CTF debug data arena */
220 static struct modctl *kobj_modules = NULL; /* modules loaded */
221 int kobj_mmu_pagesize; /* system pagesize */
222 static int lg_pagesize; /* "large" pagesize */
223 static int kobj_last_module_id = 0; /* id assignment */
224 static kmutex_t kobj_lock; /* protects mach memory list */
225
226 /*
227 * The following functions have been implemented by the kernel.
228 * However, many 3rd party drivers provide their own implementations
229 * of these functions. When such drivers are loaded, messages
230 * indicating that these symbols have been multiply defined will be
231 * emitted to the console. To avoid alarming customers for no good
232 * reason, we simply suppress such warnings for the following set of
233 * functions.
234 */
235 static char *suppress_sym_list[] =
236 {
237 "strstr",
238 "strncat",
239 "strlcat",
240 "strlcpy",
241 "strspn",
242 "memcpy",
243 "memset",
244 "memmove",
245 "memcmp",
246 "memchr",
247 "__udivdi3",
248 "__divdi3",
249 "__umoddi3",
250 "__moddi3",
251 NULL /* This entry must exist */
252 };
253
254 /* indexed by KOBJ_NOTIFY_* */
255 static kobj_notify_list_t *kobj_notifiers[KOBJ_NOTIFY_MAX + 1];
256
257 /*
258 * Prefix for statically defined tracing (SDT) DTrace probes.
259 */
260 const char *sdt_prefix = "__dtrace_probe_";
261
262 /*
263 * Beginning and end of the kernel's dynamic text/data segments.
264 */
265 static caddr_t _text;
266 static caddr_t _etext;
267 static caddr_t _data;
268
269 /*
270 * The sparc linker doesn't create a memory location
271 * for a variable named _edata, so _edata can only be
272 * referred to, not modified. krtld needs a static
273 * variable to modify it - within krtld, of course -
274 * outside of krtld, e_data is used in all kernels.
275 */
276 #if defined(__sparc)
277 static caddr_t _edata;
278 #else
279 extern caddr_t _edata;
280 #endif
281
282 Addr dynseg = 0; /* load address of "dynamic" segment */
283 size_t dynsize; /* "dynamic" segment size */
284
285
286 int standalone = 1; /* an unwholey kernel? */
287 int use_iflush; /* iflush after relocations */
288
289 /*
290 * _kobj_printf() and _vkobj_printf()
291 *
292 * Common printf function pointer. Can handle only one conversion
293 * specification in the format string. Some of the functions invoked
294 * through this function pointer cannot handle more that one conversion
295 * specification in the format string.
296 */
297 void (*_kobj_printf)(void *, const char *, ...) __KPRINTFLIKE(2);
298 void (*_vkobj_printf)(void *, const char *, va_list) __KVPRINTFLIKE(2);
299
300 /*
301 * Standalone function pointers for use within krtld.
302 * Many platforms implement optimized platmod versions of
303 * utilities such as bcopy and any such are not yet available
304 * until the kernel is more completely stitched together.
305 * See kobj_impl.h
306 */
307 void (*kobj_bcopy)(const void *, void *, size_t);
308 void (*kobj_bzero)(void *, size_t);
309 size_t (*kobj_strlcat)(char *, const char *, size_t);
310
311 static kobj_stat_t kobj_stat;
312
313 #define MINALIGN 8 /* at least a double-word */
314
315 int
get_weakish_int(int * ip)316 get_weakish_int(int *ip)
317 {
318 if (standalone)
319 return (0);
320 return (ip == NULL ? 0 : *ip);
321 }
322
323 static void *
get_weakish_pointer(void ** ptrp)324 get_weakish_pointer(void **ptrp)
325 {
326 if (standalone)
327 return (0);
328 return (ptrp == NULL ? 0 : *ptrp);
329 }
330
331 /*
332 * XXX fix dependencies on "kernel"; this should work
333 * for other standalone binaries as well.
334 *
335 * XXX Fix hashing code to use one pointer to
336 * hash entries.
337 * |----------|
338 * | nbuckets |
339 * |----------|
340 * | nchains |
341 * |----------|
342 * | bucket[] |
343 * |----------|
344 * | chain[] |
345 * |----------|
346 */
347
348 /*
349 * Load, bind and relocate all modules that
350 * form the primary kernel. At this point, our
351 * externals have not been relocated.
352 */
353 void
kobj_init(void * romvec,void * dvec,struct bootops * bootvec,val_t * bootaux)354 kobj_init(
355 void *romvec,
356 void *dvec,
357 struct bootops *bootvec,
358 val_t *bootaux)
359 {
360 struct module *mp;
361 struct modctl *modp;
362 Addr entry;
363 char filename[MAXPATHLEN];
364
365 /*
366 * Save these to pass on to
367 * the booted standalone.
368 */
369 romp = romvec;
370 dbvec = dvec;
371
372 ops = bootvec;
373 kobj_setup_standalone_vectors();
374
375 KOBJ_MARK("Entered kobj_init()");
376
377 (void) BOP_GETPROP(ops, "whoami", filename);
378
379 /*
380 * We don't support standalone debuggers anymore. The use of kadb
381 * will interfere with the later use of kmdb. Let the user mend
382 * their ways now. Users will reach this message if they still
383 * have the kadb binary on their system (perhaps they used an old
384 * bfu, or maybe they intentionally copied it there) and have
385 * specified its use in a way that eluded our checking in the boot
386 * program.
387 */
388 if (dvec != NULL) {
389 _kobj_printf(ops, "\nWARNING: Standalone debuggers such as "
390 "kadb are no longer supported\n\n");
391 goto fail;
392 }
393
394 #if defined(_OBP)
395 /*
396 * OBP allows us to read both the ramdisk and
397 * the underlying root fs when root is a disk.
398 * This can lower incidences of unbootable systems
399 * when the archive is out-of-date with the /etc
400 * state files.
401 */
402 if (BOP_MOUNTROOT() != BOOT_SVC_OK) {
403 _kobj_printf(ops, "can't mount boot fs\n");
404 goto fail;
405 }
406 #else
407 /* on x86, we always boot with a ramdisk */
408 if (kobj_boot_mountroot() != 0) {
409 goto fail;
410 }
411
412 /*
413 * Now that the ramdisk is mounted, finish boot property
414 * initialization.
415 */
416 read_bootenvrc();
417
418 #if !defined(_UNIX_KRTLD)
419 /*
420 * 'unix' is linked together with 'krtld' into one executable and
421 * the early boot code does -not- hand us any of the dynamic metadata
422 * about the executable. In particular, it does not read in, map or
423 * otherwise look at the program headers. We fake all that up now.
424 *
425 * We do this early as DTrace static probes call undefined references.
426 * We have to process those relocations before calling any of them.
427 *
428 * OBP tells kobj_start() where the ELF image is in memory, so it
429 * synthesized bootaux before kobj_init() was called
430 */
431 if (bootaux[BA_PHDR].ba_ptr == NULL)
432 synthetic_bootaux(filename, bootaux);
433
434 #endif /* !_UNIX_KRTLD */
435 #endif /* _OBP */
436
437 /*
438 * Save the interesting attribute-values
439 * (scanned by kobj_boot).
440 */
441 attr_val(bootaux);
442
443 /*
444 * Set the module search path.
445 */
446 kobj_module_path = getmodpath(filename);
447
448 boot_cpu_compatible_list = find_libmacro("CPU");
449
450 /*
451 * These two modules have actually been
452 * loaded by boot, but we finish the job
453 * by introducing them into the world of
454 * loadable modules.
455 */
456
457 mp = load_exec(bootaux, filename);
458 load_linker(bootaux);
459
460 /*
461 * Load all the primary dependent modules.
462 */
463 if (load_primary(mp, KOBJ_LM_PRIMARY) == -1)
464 goto fail;
465
466 /*
467 * Glue it together.
468 */
469 if (bind_primary(bootaux, KOBJ_LM_PRIMARY) == -1)
470 goto fail;
471
472 entry = bootaux[BA_ENTRY].ba_val;
473
474 /*
475 * Get the boot flags
476 */
477 bootflags(ops);
478
479 if (boothowto & RB_VERBOSE)
480 kobj_lm_dump(KOBJ_LM_PRIMARY);
481
482 kobj_kdi_init();
483
484 if (boothowto & RB_KMDB) {
485 if (load_kmdb(bootaux) < 0)
486 goto fail;
487 }
488
489 /*
490 * Post setup.
491 */
492 s_text = _text;
493 e_text = _etext;
494 s_data = _data;
495 e_data = _edata;
496
497 kobj_sync_instruction_memory(s_text, e_text - s_text);
498
499 #ifdef KOBJ_DEBUG
500 if (kobj_debug & D_DEBUG)
501 _kobj_printf(ops,
502 "krtld: transferring control to: 0x%lx\n", entry);
503 #endif
504
505 /*
506 * Make sure the mod system knows about the modules already loaded.
507 */
508 last_module_id = kobj_last_module_id;
509 bcopy(kobj_modules, &modules, sizeof (modules));
510 modp = &modules;
511 do {
512 if (modp->mod_next == kobj_modules)
513 modp->mod_next = &modules;
514 if (modp->mod_prev == kobj_modules)
515 modp->mod_prev = &modules;
516 } while ((modp = modp->mod_next) != &modules);
517
518 standalone = 0;
519
520 #ifdef KOBJ_DEBUG
521 if (kobj_debug & D_DEBUG)
522 _kobj_printf(ops,
523 "krtld: really transferring control to: 0x%lx\n", entry);
524 #endif
525
526 /* restore printf/bcopy/bzero vectors before returning */
527 kobj_restore_vectors();
528
529 #if defined(_DBOOT)
530 /*
531 * krtld was called from a dboot ELF section, the embedded
532 * dboot code contains the real entry via bootaux
533 */
534 exitto((caddr_t)entry);
535 #else
536 /*
537 * krtld was directly called from startup
538 */
539 return;
540 #endif
541
542 fail:
543
544 _kobj_printf(ops, "krtld: error during initial load/link phase\n");
545
546 #if !defined(_UNIX_KRTLD)
547 _kobj_printf(ops, "\n");
548 _kobj_printf(ops, "krtld could neither locate nor resolve symbols"
549 " for:\n");
550 _kobj_printf(ops, " %s\n", filename);
551 _kobj_printf(ops, "in the boot archive. Please verify that this"
552 " file\n");
553 _kobj_printf(ops, "matches what is found in the boot archive.\n");
554 _kobj_printf(ops, "You may need to boot using the Solaris failsafe to"
555 " fix this.\n");
556 bop_panic("Unable to boot");
557 #endif
558 }
559
560 #if !defined(_UNIX_KRTLD) && !defined(_OBP)
561 /*
562 * Synthesize additional metadata that describes the executable if
563 * krtld's caller didn't do it.
564 *
565 * (When the dynamic executable has an interpreter, the boot program
566 * does all this for us. Where we don't have an interpreter, (or a
567 * even a boot program, perhaps) we have to do this for ourselves.)
568 */
569 static void
synthetic_bootaux(char * filename,val_t * bootaux)570 synthetic_bootaux(char *filename, val_t *bootaux)
571 {
572 Ehdr ehdr;
573 caddr_t phdrbase;
574 struct _buf *file;
575 int i, n;
576
577 /*
578 * Elf header
579 */
580 KOBJ_MARK("synthetic_bootaux()");
581 KOBJ_MARK(filename);
582 file = kobj_open_file(filename);
583 if (file == (struct _buf *)-1) {
584 _kobj_printf(ops, "krtld: failed to open '%s'\n", filename);
585 return;
586 }
587 KOBJ_MARK("reading program headers");
588 if (kobj_read_file(file, (char *)&ehdr, sizeof (ehdr), 0) < 0) {
589 _kobj_printf(ops, "krtld: %s: failed to read ehder\n",
590 filename);
591 return;
592 }
593
594 /*
595 * Program headers
596 */
597 bootaux[BA_PHNUM].ba_val = ehdr.e_phnum;
598 bootaux[BA_PHENT].ba_val = ehdr.e_phentsize;
599 n = ehdr.e_phentsize * ehdr.e_phnum;
600
601 phdrbase = kobj_alloc(n, KM_WAIT | KM_TMP);
602
603 if (kobj_read_file(file, phdrbase, n, ehdr.e_phoff) < 0) {
604 _kobj_printf(ops, "krtld: %s: failed to read phdrs\n",
605 filename);
606 return;
607 }
608 bootaux[BA_PHDR].ba_ptr = phdrbase;
609 kobj_close_file(file);
610 KOBJ_MARK("closed file");
611
612 /*
613 * Find the dynamic section address
614 */
615 for (i = 0; i < ehdr.e_phnum; i++) {
616 Phdr *phdr = (Phdr *)(phdrbase + ehdr.e_phentsize * i);
617
618 if (phdr->p_type == PT_DYNAMIC) {
619 bootaux[BA_DYNAMIC].ba_ptr = (void *)phdr->p_vaddr;
620 break;
621 }
622 }
623 KOBJ_MARK("synthetic_bootaux() done");
624 }
625 #endif /* !_UNIX_KRTLD && !_OBP */
626
627 /*
628 * Set up any global information derived
629 * from attribute/values in the boot or
630 * aux vector.
631 */
632 static void
attr_val(val_t * bootaux)633 attr_val(val_t *bootaux)
634 {
635 Phdr *phdr;
636 int phnum, phsize;
637 int i;
638
639 KOBJ_MARK("attr_val()");
640 kobj_mmu_pagesize = bootaux[BA_PAGESZ].ba_val;
641 lg_pagesize = bootaux[BA_LPAGESZ].ba_val;
642 use_iflush = bootaux[BA_IFLUSH].ba_val;
643
644 phdr = (Phdr *)bootaux[BA_PHDR].ba_ptr;
645 phnum = bootaux[BA_PHNUM].ba_val;
646 phsize = bootaux[BA_PHENT].ba_val;
647 for (i = 0; i < phnum; i++) {
648 phdr = (Phdr *)(bootaux[BA_PHDR].ba_val + i * phsize);
649
650 if (phdr->p_type != PT_LOAD) {
651 continue;
652 }
653 /*
654 * Bounds of the various segments.
655 */
656 if (!(phdr->p_flags & PF_X)) {
657 #if defined(_RELSEG)
658 /*
659 * sparc kernel puts the dynamic info
660 * into a separate segment, which is
661 * free'd in bop_fini()
662 */
663 ASSERT(phdr->p_vaddr != 0);
664 dynseg = phdr->p_vaddr;
665 dynsize = phdr->p_memsz;
666 #else
667 ASSERT(phdr->p_vaddr == 0);
668 #endif
669 } else {
670 if (phdr->p_flags & PF_W) {
671 _data = (caddr_t)phdr->p_vaddr;
672 _edata = _data + phdr->p_memsz;
673 } else {
674 _text = (caddr_t)phdr->p_vaddr;
675 _etext = _text + phdr->p_memsz;
676 }
677 }
678 }
679
680 /* To do the kobj_alloc, _edata needs to be set. */
681 for (i = 0; i < NLIBMACROS; i++) {
682 if (bootaux[libmacros[i].lmi_ba_index].ba_ptr != NULL) {
683 libmacros[i].lmi_list = kobj_alloc(
684 strlen(bootaux[libmacros[i].lmi_ba_index].ba_ptr) +
685 1, KM_WAIT);
686 (void) strcpy(libmacros[i].lmi_list,
687 bootaux[libmacros[i].lmi_ba_index].ba_ptr);
688 }
689 libmacros[i].lmi_macrolen = strlen(libmacros[i].lmi_macroname);
690 }
691 }
692
693 /*
694 * Set up the booted executable.
695 */
696 static struct module *
load_exec(val_t * bootaux,char * filename)697 load_exec(val_t *bootaux, char *filename)
698 {
699 struct modctl *cp;
700 struct module *mp;
701 Dyn *dyn;
702 Sym *sp;
703 int i, lsize, osize, nsize, allocsize;
704 char *libname, *tmp;
705 char path[MAXPATHLEN];
706
707 #ifdef KOBJ_DEBUG
708 if (kobj_debug & D_DEBUG)
709 _kobj_printf(ops, "module path '%s'\n", kobj_module_path);
710 #endif
711
712 KOBJ_MARK("add_primary");
713 cp = add_primary(filename, KOBJ_LM_PRIMARY);
714
715 KOBJ_MARK("struct module");
716 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
717 cp->mod_mp = mp;
718
719 /*
720 * We don't have the following information
721 * since this module is an executable and not
722 * a relocatable .o.
723 */
724 mp->symtbl_section = 0;
725 mp->shdrs = NULL;
726 mp->strhdr = NULL;
727
728 /*
729 * Since this module is the only exception,
730 * we cons up some section headers.
731 */
732 KOBJ_MARK("symhdr");
733 mp->symhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
734
735 KOBJ_MARK("strhdr");
736 mp->strhdr = kobj_zalloc(sizeof (Shdr), KM_WAIT);
737
738 mp->symhdr->sh_type = SHT_SYMTAB;
739 mp->strhdr->sh_type = SHT_STRTAB;
740 /*
741 * Scan the dynamic structure.
742 */
743 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
744 dyn->d_tag != DT_NULL; dyn++) {
745 switch (dyn->d_tag) {
746 case DT_SYMTAB:
747 mp->symspace = mp->symtbl = (char *)dyn->d_un.d_ptr;
748 mp->symhdr->sh_addr = dyn->d_un.d_ptr;
749 break;
750 case DT_HASH:
751 mp->nsyms = *((uint_t *)dyn->d_un.d_ptr + 1);
752 mp->hashsize = *(uint_t *)dyn->d_un.d_ptr;
753 break;
754 case DT_STRTAB:
755 mp->strings = (char *)dyn->d_un.d_ptr;
756 mp->strhdr->sh_addr = dyn->d_un.d_ptr;
757 break;
758 case DT_STRSZ:
759 mp->strhdr->sh_size = dyn->d_un.d_val;
760 break;
761 case DT_SYMENT:
762 mp->symhdr->sh_entsize = dyn->d_un.d_val;
763 break;
764 }
765 }
766
767 /*
768 * Collapse any DT_NEEDED entries into one string.
769 */
770 nsize = osize = 0;
771 allocsize = MAXPATHLEN;
772
773 KOBJ_MARK("depends_on");
774 mp->depends_on = kobj_alloc(allocsize, KM_WAIT);
775
776 for (dyn = (Dyn *) bootaux[BA_DYNAMIC].ba_ptr;
777 dyn->d_tag != DT_NULL; dyn++)
778 if (dyn->d_tag == DT_NEEDED) {
779 char *_lib;
780
781 libname = mp->strings + dyn->d_un.d_val;
782 if (strchr(libname, '$') != NULL) {
783 if ((_lib = expand_libmacro(libname,
784 path, path)) != NULL)
785 libname = _lib;
786 else
787 _kobj_printf(ops, "krtld: "
788 "load_exec: fail to "
789 "expand %s\n", libname);
790 }
791 lsize = strlen(libname);
792 nsize += lsize;
793 if (nsize + 1 > allocsize) {
794 KOBJ_MARK("grow depends_on");
795 tmp = kobj_alloc(allocsize + MAXPATHLEN,
796 KM_WAIT);
797 bcopy(mp->depends_on, tmp, osize);
798 kobj_free(mp->depends_on, allocsize);
799 mp->depends_on = tmp;
800 allocsize += MAXPATHLEN;
801 }
802 bcopy(libname, mp->depends_on + osize, lsize);
803 *(mp->depends_on + nsize) = ' '; /* separate */
804 nsize++;
805 osize = nsize;
806 }
807 if (nsize) {
808 mp->depends_on[nsize - 1] = '\0'; /* terminate the string */
809 /*
810 * alloc with exact size and copy whatever it got over
811 */
812 KOBJ_MARK("realloc depends_on");
813 tmp = kobj_alloc(nsize, KM_WAIT);
814 bcopy(mp->depends_on, tmp, nsize);
815 kobj_free(mp->depends_on, allocsize);
816 mp->depends_on = tmp;
817 } else {
818 kobj_free(mp->depends_on, allocsize);
819 mp->depends_on = NULL;
820 }
821
822 mp->flags = KOBJ_EXEC|KOBJ_PRIM; /* NOT a relocatable .o */
823 mp->symhdr->sh_size = mp->nsyms * mp->symhdr->sh_entsize;
824 /*
825 * We allocate our own table since we don't
826 * hash undefined references.
827 */
828 KOBJ_MARK("chains");
829 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
830 KOBJ_MARK("buckets");
831 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
832
833 mp->text = _text;
834 mp->data = _data;
835
836 mp->text_size = _etext - _text;
837 mp->data_size = _edata - _data;
838
839 cp->mod_text = mp->text;
840 cp->mod_text_size = mp->text_size;
841
842 mp->filename = cp->mod_filename;
843
844 #ifdef KOBJ_DEBUG
845 if (kobj_debug & D_LOADING) {
846 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
847 _kobj_printf(ops, "\ttext: 0x%p", mp->text);
848 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
849 _kobj_printf(ops, "\tdata: 0x%p", mp->data);
850 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
851 }
852 #endif /* KOBJ_DEBUG */
853
854 /*
855 * Insert symbols into the hash table.
856 */
857 for (i = 0; i < mp->nsyms; i++) {
858 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
859
860 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
861 continue;
862 #if defined(__sparc)
863 /*
864 * Register symbols are ignored in the kernel
865 */
866 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER)
867 continue;
868 #endif /* __sparc */
869
870 sym_insert(mp, mp->strings + sp->st_name, i);
871 }
872
873 KOBJ_MARK("load_exec done");
874 return (mp);
875 }
876
877 /*
878 * Set up the linker module (if it's compiled in, LDNAME is NULL)
879 */
880 static void
load_linker(val_t * bootaux)881 load_linker(val_t *bootaux)
882 {
883 struct module *kmp = (struct module *)kobj_modules->mod_mp;
884 struct module *mp;
885 struct modctl *cp;
886 int i;
887 Shdr *shp;
888 Sym *sp;
889 int shsize;
890 char *dlname = (char *)bootaux[BA_LDNAME].ba_ptr;
891
892 /*
893 * On some architectures, krtld is compiled into the kernel.
894 */
895 if (dlname == NULL)
896 return;
897
898 cp = add_primary(dlname, KOBJ_LM_PRIMARY);
899
900 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
901
902 cp->mod_mp = mp;
903 mp->hdr = *(Ehdr *)bootaux[BA_LDELF].ba_ptr;
904 shsize = mp->hdr.e_shentsize * mp->hdr.e_shnum;
905 mp->shdrs = kobj_alloc(shsize, KM_WAIT);
906 bcopy(bootaux[BA_LDSHDR].ba_ptr, mp->shdrs, shsize);
907
908 for (i = 1; i < (int)mp->hdr.e_shnum; i++) {
909 shp = (Shdr *)(mp->shdrs + (i * mp->hdr.e_shentsize));
910
911 if (shp->sh_flags & SHF_ALLOC) {
912 if (shp->sh_flags & SHF_WRITE) {
913 if (mp->data == NULL)
914 mp->data = (char *)shp->sh_addr;
915 } else if (mp->text == NULL) {
916 mp->text = (char *)shp->sh_addr;
917 }
918 }
919 if (shp->sh_type == SHT_SYMTAB) {
920 mp->symtbl_section = i;
921 mp->symhdr = shp;
922 mp->symspace = mp->symtbl = (char *)shp->sh_addr;
923 }
924 }
925 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
926 mp->flags = KOBJ_INTERP|KOBJ_PRIM;
927 mp->strhdr = (Shdr *)
928 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
929 mp->strings = (char *)mp->strhdr->sh_addr;
930 mp->hashsize = kobj_gethashsize(mp->nsyms);
931
932 mp->symsize = mp->symhdr->sh_size + mp->strhdr->sh_size + sizeof (int) +
933 (mp->hashsize + mp->nsyms) * sizeof (symid_t);
934
935 mp->chains = kobj_zalloc(mp->nsyms * sizeof (symid_t), KM_WAIT);
936 mp->buckets = kobj_zalloc(mp->hashsize * sizeof (symid_t), KM_WAIT);
937
938 mp->bss = bootaux[BA_BSS].ba_val;
939 mp->bss_align = 0; /* pre-aligned during allocation */
940 mp->bss_size = (uintptr_t)_edata - mp->bss;
941 mp->text_size = _etext - mp->text;
942 mp->data_size = _edata - mp->data;
943 mp->filename = cp->mod_filename;
944 cp->mod_text = mp->text;
945 cp->mod_text_size = mp->text_size;
946
947 /*
948 * Now that we've figured out where the linker is,
949 * set the limits for the booted object.
950 */
951 kmp->text_size = (size_t)(mp->text - kmp->text);
952 kmp->data_size = (size_t)(mp->data - kmp->data);
953 kobj_modules->mod_text_size = kmp->text_size;
954
955 #ifdef KOBJ_DEBUG
956 if (kobj_debug & D_LOADING) {
957 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
958 _kobj_printf(ops, "\ttext:0x%p", mp->text);
959 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
960 _kobj_printf(ops, "\tdata:0x%p", mp->data);
961 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
962 }
963 #endif /* KOBJ_DEBUG */
964
965 /*
966 * Insert the symbols into the hash table.
967 */
968 for (i = 0; i < mp->nsyms; i++) {
969 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
970
971 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
972 continue;
973 if (ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
974 if (sp->st_shndx == SHN_COMMON)
975 sp->st_shndx = SHN_ABS;
976 }
977 sym_insert(mp, mp->strings + sp->st_name, i);
978 }
979
980 }
981
982 static kobj_notify_list_t **
kobj_notify_lookup(uint_t type)983 kobj_notify_lookup(uint_t type)
984 {
985 ASSERT(type != 0 && type < sizeof (kobj_notifiers) /
986 sizeof (kobj_notify_list_t *));
987
988 return (&kobj_notifiers[type]);
989 }
990
991 int
kobj_notify_add(kobj_notify_list_t * knp)992 kobj_notify_add(kobj_notify_list_t *knp)
993 {
994 kobj_notify_list_t **knl;
995
996 knl = kobj_notify_lookup(knp->kn_type);
997
998 knp->kn_next = NULL;
999 knp->kn_prev = NULL;
1000
1001 mutex_enter(&kobj_lock);
1002
1003 if (*knl != NULL) {
1004 (*knl)->kn_prev = knp;
1005 knp->kn_next = *knl;
1006 }
1007 (*knl) = knp;
1008
1009 mutex_exit(&kobj_lock);
1010 return (0);
1011 }
1012
1013 int
kobj_notify_remove(kobj_notify_list_t * knp)1014 kobj_notify_remove(kobj_notify_list_t *knp)
1015 {
1016 kobj_notify_list_t **knl = kobj_notify_lookup(knp->kn_type);
1017 kobj_notify_list_t *tknp;
1018
1019 mutex_enter(&kobj_lock);
1020
1021 if ((tknp = knp->kn_next) != NULL)
1022 tknp->kn_prev = knp->kn_prev;
1023
1024 if ((tknp = knp->kn_prev) != NULL)
1025 tknp->kn_next = knp->kn_next;
1026 else
1027 *knl = knp->kn_next;
1028
1029 mutex_exit(&kobj_lock);
1030
1031 return (0);
1032 }
1033
1034 /*
1035 * Notify all interested callbacks of a specified change in module state.
1036 */
1037 static void
kobj_notify(int type,struct modctl * modp)1038 kobj_notify(int type, struct modctl *modp)
1039 {
1040 kobj_notify_list_t *knp;
1041
1042 if (modp->mod_loadflags & MOD_NONOTIFY || standalone)
1043 return;
1044
1045 mutex_enter(&kobj_lock);
1046
1047 for (knp = *(kobj_notify_lookup(type)); knp != NULL; knp = knp->kn_next)
1048 knp->kn_func(type, modp);
1049
1050 /*
1051 * KDI notification must be last (it has to allow for work done by the
1052 * other notification callbacks), so we call it manually.
1053 */
1054 kobj_kdi_mod_notify(type, modp);
1055
1056 mutex_exit(&kobj_lock);
1057 }
1058
1059 /*
1060 * Create the module path.
1061 */
1062 static char *
getmodpath(const char * filename)1063 getmodpath(const char *filename)
1064 {
1065 char *path = kobj_zalloc(MAXPATHLEN, KM_WAIT);
1066
1067 /*
1068 * Platform code gets first crack, then add
1069 * the default components
1070 */
1071 mach_modpath(path, filename);
1072 if (*path != '\0')
1073 (void) strcat(path, " ");
1074 return (strcat(path, MOD_DEFPATH));
1075 }
1076
1077 static struct modctl *
add_primary(const char * filename,int lmid)1078 add_primary(const char *filename, int lmid)
1079 {
1080 struct modctl *cp;
1081
1082 cp = kobj_zalloc(sizeof (struct modctl), KM_WAIT);
1083
1084 cp->mod_filename = kobj_alloc(strlen(filename) + 1, KM_WAIT);
1085
1086 /*
1087 * For symbol lookup, we assemble our own
1088 * modctl list of the primary modules.
1089 */
1090
1091 (void) strcpy(cp->mod_filename, filename);
1092 cp->mod_modname = basename(cp->mod_filename);
1093
1094 /* set values for modinfo assuming that the load will work */
1095 cp->mod_prim = 1;
1096 cp->mod_loaded = 1;
1097 cp->mod_installed = 1;
1098 cp->mod_loadcnt = 1;
1099 cp->mod_loadflags = MOD_NOAUTOUNLOAD;
1100
1101 cp->mod_id = kobj_last_module_id++;
1102
1103 /*
1104 * Link the module in. We'll pass this info on
1105 * to the mod squad later.
1106 */
1107 if (kobj_modules == NULL) {
1108 kobj_modules = cp;
1109 cp->mod_prev = cp->mod_next = cp;
1110 } else {
1111 cp->mod_prev = kobj_modules->mod_prev;
1112 cp->mod_next = kobj_modules;
1113 kobj_modules->mod_prev->mod_next = cp;
1114 kobj_modules->mod_prev = cp;
1115 }
1116
1117 kobj_lm_append(lmid, cp);
1118
1119 return (cp);
1120 }
1121
1122 static int
bind_primary(val_t * bootaux,int lmid)1123 bind_primary(val_t *bootaux, int lmid)
1124 {
1125 struct modctl_list *linkmap = kobj_lm_lookup(lmid);
1126 struct modctl_list *lp;
1127 struct module *mp;
1128
1129 /*
1130 * Do common symbols.
1131 */
1132 for (lp = linkmap; lp; lp = lp->modl_next) {
1133 mp = mod(lp);
1134
1135 /*
1136 * Don't do common section relocations for modules that
1137 * don't need it.
1138 */
1139 if (mp->flags & (KOBJ_EXEC|KOBJ_INTERP))
1140 continue;
1141
1142 if (do_common(mp) < 0)
1143 return (-1);
1144 }
1145
1146 /*
1147 * Resolve symbols.
1148 */
1149 for (lp = linkmap; lp; lp = lp->modl_next) {
1150 mp = mod(lp);
1151
1152 if (do_symbols(mp, 0) < 0)
1153 return (-1);
1154 }
1155
1156 /*
1157 * Do relocations.
1158 */
1159 for (lp = linkmap; lp; lp = lp->modl_next) {
1160 mp = mod(lp);
1161
1162 if (mp->flags & KOBJ_EXEC) {
1163 Dyn *dyn;
1164 Word relasz = 0, relaent = 0;
1165 char *rela = NULL;
1166
1167 for (dyn = (Dyn *)bootaux[BA_DYNAMIC].ba_ptr;
1168 dyn->d_tag != DT_NULL; dyn++) {
1169 switch (dyn->d_tag) {
1170 case DT_RELASZ:
1171 case DT_RELSZ:
1172 relasz = dyn->d_un.d_val;
1173 break;
1174 case DT_RELAENT:
1175 case DT_RELENT:
1176 relaent = dyn->d_un.d_val;
1177 break;
1178 case DT_RELA:
1179 rela = (char *)dyn->d_un.d_ptr;
1180 break;
1181 case DT_REL:
1182 rela = (char *)dyn->d_un.d_ptr;
1183 break;
1184 }
1185 }
1186 if (relasz == 0 ||
1187 relaent == 0 || rela == NULL) {
1188 _kobj_printf(ops, "krtld: bind_primary(): "
1189 "no relocation information found for "
1190 "module %s\n", mp->filename);
1191 return (-1);
1192 }
1193 #ifdef KOBJ_DEBUG
1194 if (kobj_debug & D_RELOCATIONS)
1195 _kobj_printf(ops, "krtld: relocating: file=%s "
1196 "KOBJ_EXEC\n", mp->filename);
1197 #endif
1198 if (do_relocate(mp, rela, relasz/relaent, relaent,
1199 (Addr)mp->text) < 0)
1200 return (-1);
1201 } else {
1202 if (do_relocations(mp) < 0)
1203 return (-1);
1204 }
1205
1206 kobj_sync_instruction_memory(mp->text, mp->text_size);
1207 }
1208
1209 for (lp = linkmap; lp; lp = lp->modl_next) {
1210 mp = mod(lp);
1211
1212 /*
1213 * We need to re-read the full symbol table for the boot file,
1214 * since we couldn't use the full one before. We also need to
1215 * load the CTF sections of both the boot file and the
1216 * interpreter (us).
1217 */
1218 if (mp->flags & KOBJ_EXEC) {
1219 struct _buf *file;
1220 int n;
1221
1222 file = kobj_open_file(mp->filename);
1223 if (file == (struct _buf *)-1)
1224 return (-1);
1225 if (kobj_read_file(file, (char *)&mp->hdr,
1226 sizeof (mp->hdr), 0) < 0)
1227 return (-1);
1228 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1229 mp->shdrs = kobj_alloc(n, KM_WAIT);
1230 if (kobj_read_file(file, mp->shdrs, n,
1231 mp->hdr.e_shoff) < 0)
1232 return (-1);
1233 if (get_syms(mp, file) < 0)
1234 return (-1);
1235 if (get_ctf(mp, file) < 0)
1236 return (-1);
1237 kobj_close_file(file);
1238 mp->flags |= KOBJ_RELOCATED;
1239
1240 } else if (mp->flags & KOBJ_INTERP) {
1241 struct _buf *file;
1242
1243 /*
1244 * The interpreter path fragment in mp->filename
1245 * will already have the module directory suffix
1246 * in it (if appropriate).
1247 */
1248 file = kobj_open_path(mp->filename, 1, 0);
1249 if (file == (struct _buf *)-1)
1250 return (-1);
1251 if (get_ctf(mp, file) < 0)
1252 return (-1);
1253 kobj_close_file(file);
1254 mp->flags |= KOBJ_RELOCATED;
1255 }
1256 }
1257
1258 return (0);
1259 }
1260
1261 static struct modctl *
mod_already_loaded(char * modname)1262 mod_already_loaded(char *modname)
1263 {
1264 struct modctl *mctl = kobj_modules;
1265
1266 do {
1267 if (strcmp(modname, mctl->mod_filename) == 0)
1268 return (mctl);
1269 mctl = mctl->mod_next;
1270
1271 } while (mctl != kobj_modules);
1272
1273 return (NULL);
1274 }
1275
1276 /*
1277 * Load all the primary dependent modules.
1278 */
1279 static int
load_primary(struct module * mp,int lmid)1280 load_primary(struct module *mp, int lmid)
1281 {
1282 struct modctl *cp;
1283 struct module *dmp;
1284 char *p, *q;
1285 char modname[MODMAXNAMELEN];
1286
1287 if ((p = mp->depends_on) == NULL)
1288 return (0);
1289
1290 /* CONSTANTCONDITION */
1291 while (1) {
1292 /*
1293 * Skip space.
1294 */
1295 while (*p && (*p == ' ' || *p == '\t'))
1296 p++;
1297 /*
1298 * Get module name.
1299 */
1300 q = modname;
1301 while (*p && *p != ' ' && *p != '\t')
1302 *q++ = *p++;
1303
1304 if (q == modname)
1305 break;
1306
1307 *q = '\0';
1308 /*
1309 * Check for dup dependencies.
1310 */
1311 if (strcmp(modname, "dtracestubs") == 0 ||
1312 mod_already_loaded(modname) != NULL)
1313 continue;
1314
1315 cp = add_primary(modname, lmid);
1316 cp->mod_busy = 1;
1317 /*
1318 * Load it.
1319 */
1320 (void) kobj_load_module(cp, 1);
1321 cp->mod_busy = 0;
1322
1323 if ((dmp = cp->mod_mp) == NULL) {
1324 cp->mod_loaded = 0;
1325 cp->mod_installed = 0;
1326 cp->mod_loadcnt = 0;
1327 return (-1);
1328 }
1329
1330 add_dependent(mp, dmp);
1331 dmp->flags |= KOBJ_PRIM;
1332
1333 /*
1334 * Recurse.
1335 */
1336 if (load_primary(dmp, lmid) == -1) {
1337 cp->mod_loaded = 0;
1338 cp->mod_installed = 0;
1339 cp->mod_loadcnt = 0;
1340 return (-1);
1341 }
1342 }
1343 return (0);
1344 }
1345
1346 static int
console_is_usb_serial(void)1347 console_is_usb_serial(void)
1348 {
1349 char *console;
1350 int len, ret;
1351
1352 if ((len = BOP_GETPROPLEN(ops, "console")) == -1)
1353 return (0);
1354
1355 console = kobj_zalloc(len, KM_WAIT|KM_TMP);
1356 (void) BOP_GETPROP(ops, "console", console);
1357 ret = (strcmp(console, "usb-serial") == 0);
1358 kobj_free(console, len);
1359
1360 return (ret);
1361 }
1362
1363 static int
load_kmdb(val_t * bootaux)1364 load_kmdb(val_t *bootaux)
1365 {
1366 struct modctl *mctl;
1367 struct module *mp;
1368 Sym *sym;
1369
1370 if (console_is_usb_serial()) {
1371 _kobj_printf(ops, "kmdb not loaded "
1372 "(unsupported on usb serial console)\n");
1373 return (0);
1374 }
1375
1376 _kobj_printf(ops, "Loading kmdb...\n");
1377
1378 if ((mctl = add_primary("misc/kmdbmod", KOBJ_LM_DEBUGGER)) == NULL)
1379 return (-1);
1380
1381 mctl->mod_busy = 1;
1382 (void) kobj_load_module(mctl, 1);
1383 mctl->mod_busy = 0;
1384
1385 if ((mp = mctl->mod_mp) == NULL)
1386 return (-1);
1387
1388 mp->flags |= KOBJ_PRIM;
1389
1390 if (load_primary(mp, KOBJ_LM_DEBUGGER) < 0)
1391 return (-1);
1392
1393 if (boothowto & RB_VERBOSE)
1394 kobj_lm_dump(KOBJ_LM_DEBUGGER);
1395
1396 if (bind_primary(bootaux, KOBJ_LM_DEBUGGER) < 0)
1397 return (-1);
1398
1399 if ((sym = lookup_one(mctl->mod_mp, "kctl_boot_activate")) == NULL)
1400 return (-1);
1401
1402 #ifdef KOBJ_DEBUG
1403 if (kobj_debug & D_DEBUG) {
1404 _kobj_printf(ops, "calling kctl_boot_activate() @ 0x%lx\n",
1405 sym->st_value);
1406 _kobj_printf(ops, "\tops 0x%p\n", ops);
1407 _kobj_printf(ops, "\tromp 0x%p\n", romp);
1408 }
1409 #endif
1410
1411 if (((kctl_boot_activate_f *)sym->st_value)(ops, romp, 0,
1412 (const char **)kobj_kmdb_argv) < 0)
1413 return (-1);
1414
1415 return (0);
1416 }
1417
1418 /*
1419 * Return a string listing module dependencies.
1420 */
1421 static char *
depends_on(struct module * mp)1422 depends_on(struct module *mp)
1423 {
1424 Sym *sp;
1425 char *depstr, *q;
1426
1427 /*
1428 * The module doesn't have a depends_on value, so let's try it the
1429 * old-fashioned way - via "_depends_on"
1430 */
1431 if ((sp = lookup_one(mp, "_depends_on")) == NULL)
1432 return (NULL);
1433
1434 q = (char *)sp->st_value;
1435
1436 #ifdef KOBJ_DEBUG
1437 /*
1438 * _depends_on is a deprecated interface, so we warn about its use
1439 * irrespective of subsequent processing errors. How else are we going
1440 * to be able to deco this interface completely?
1441 * Changes initially limited to DEBUG because third-party modules
1442 * should be flagged to developers before general use base.
1443 */
1444 _kobj_printf(ops,
1445 "Warning: %s uses deprecated _depends_on interface.\n",
1446 mp->filename);
1447 _kobj_printf(ops, "Please notify module developer or vendor.\n");
1448 #endif
1449
1450 /*
1451 * Idiot checks. Make sure it's
1452 * in-bounds and NULL terminated.
1453 */
1454 if (kobj_addrcheck(mp, q) || q[sp->st_size - 1] != '\0') {
1455 _kobj_printf(ops, "Error processing dependency for %s\n",
1456 mp->filename);
1457 return (NULL);
1458 }
1459
1460 depstr = (char *)kobj_alloc(strlen(q) + 1, KM_WAIT);
1461 (void) strcpy(depstr, q);
1462
1463 return (depstr);
1464 }
1465
1466 void
kobj_getmodinfo(void * xmp,struct modinfo * modinfo)1467 kobj_getmodinfo(void *xmp, struct modinfo *modinfo)
1468 {
1469 struct module *mp;
1470 mp = (struct module *)xmp;
1471
1472 modinfo->mi_base = mp->text;
1473 modinfo->mi_size = mp->text_size + mp->data_size;
1474 }
1475
1476 /*
1477 * kobj_export_ksyms() performs the following services:
1478 *
1479 * (1) Migrates the symbol table from boot/kobj memory to the ksyms arena.
1480 * (2) Removes unneeded symbols to save space.
1481 * (3) Reduces memory footprint by using VM_BESTFIT allocations.
1482 * (4) Makes the symbol table visible to /dev/ksyms.
1483 */
1484 static void
kobj_export_ksyms(struct module * mp)1485 kobj_export_ksyms(struct module *mp)
1486 {
1487 Sym *esp = (Sym *)(mp->symtbl + mp->symhdr->sh_size);
1488 Sym *sp, *osp;
1489 char *name;
1490 size_t namelen;
1491 struct module *omp;
1492 uint_t nsyms;
1493 size_t symsize = mp->symhdr->sh_entsize;
1494 size_t locals = 1;
1495 size_t strsize;
1496
1497 /*
1498 * Make a copy of the original module structure.
1499 */
1500 omp = kobj_alloc(sizeof (struct module), KM_WAIT);
1501 bcopy(mp, omp, sizeof (struct module));
1502
1503 /*
1504 * Compute the sizes of the new symbol table sections.
1505 */
1506 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1507 if (osp->st_value == 0)
1508 continue;
1509 if (sym_lookup(omp, osp) == NULL)
1510 continue;
1511 name = omp->strings + osp->st_name;
1512 namelen = strlen(name);
1513 if (ELF_ST_BIND(osp->st_info) == STB_LOCAL)
1514 locals++;
1515 nsyms++;
1516 strsize += namelen + 1;
1517 }
1518
1519 mp->nsyms = nsyms;
1520 mp->hashsize = kobj_gethashsize(mp->nsyms);
1521
1522 /*
1523 * ksyms_lock must be held as writer during any operation that
1524 * modifies ksyms_arena, including allocation from same, and
1525 * must not be dropped until the arena is vmem_walk()able.
1526 */
1527 rw_enter(&ksyms_lock, RW_WRITER);
1528
1529 /*
1530 * Allocate space for the new section headers (symtab and strtab),
1531 * symbol table, buckets, chains, and strings.
1532 */
1533 mp->symsize = (2 * sizeof (Shdr)) + (nsyms * symsize) +
1534 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + strsize;
1535
1536 if (mp->flags & KOBJ_NOKSYMS) {
1537 mp->symspace = kobj_alloc(mp->symsize, KM_WAIT);
1538 } else {
1539 mp->symspace = vmem_alloc(ksyms_arena, mp->symsize,
1540 VM_BESTFIT | VM_SLEEP);
1541 }
1542 bzero(mp->symspace, mp->symsize);
1543
1544 /*
1545 * Divvy up symspace.
1546 */
1547 mp->shdrs = mp->symspace;
1548 mp->symhdr = (Shdr *)mp->shdrs;
1549 mp->strhdr = (Shdr *)(mp->symhdr + 1);
1550 mp->symtbl = (char *)(mp->strhdr + 1);
1551 mp->buckets = (symid_t *)(mp->symtbl + (nsyms * symsize));
1552 mp->chains = (symid_t *)(mp->buckets + mp->hashsize);
1553 mp->strings = (char *)(mp->chains + nsyms);
1554
1555 /*
1556 * Fill in the new section headers (symtab and strtab).
1557 */
1558 mp->hdr.e_shnum = 2;
1559 mp->symtbl_section = 0;
1560
1561 mp->symhdr->sh_type = SHT_SYMTAB;
1562 mp->symhdr->sh_addr = (Addr)mp->symtbl;
1563 mp->symhdr->sh_size = nsyms * symsize;
1564 mp->symhdr->sh_link = 1;
1565 mp->symhdr->sh_info = locals;
1566 mp->symhdr->sh_addralign = sizeof (Addr);
1567 mp->symhdr->sh_entsize = symsize;
1568
1569 mp->strhdr->sh_type = SHT_STRTAB;
1570 mp->strhdr->sh_addr = (Addr)mp->strings;
1571 mp->strhdr->sh_size = strsize;
1572 mp->strhdr->sh_addralign = 1;
1573
1574 /*
1575 * Construct the new symbol table.
1576 */
1577 for (nsyms = strsize = 1, osp = (Sym *)omp->symtbl; osp < esp; osp++) {
1578 if (osp->st_value == 0)
1579 continue;
1580 if (sym_lookup(omp, osp) == NULL)
1581 continue;
1582 name = omp->strings + osp->st_name;
1583 namelen = strlen(name);
1584 sp = (Sym *)(mp->symtbl + symsize * nsyms);
1585 bcopy(osp, sp, symsize);
1586 bcopy(name, mp->strings + strsize, namelen);
1587 sp->st_name = strsize;
1588 sym_insert(mp, name, nsyms);
1589 nsyms++;
1590 strsize += namelen + 1;
1591 }
1592
1593 rw_exit(&ksyms_lock);
1594
1595 /*
1596 * Free the old section headers -- we'll never need them again.
1597 */
1598 if (!(mp->flags & KOBJ_PRIM)) {
1599 uint_t shn;
1600 Shdr *shp;
1601
1602 for (shn = 1; shn < omp->hdr.e_shnum; shn++) {
1603 shp = (Shdr *)(omp->shdrs + shn * omp->hdr.e_shentsize);
1604 switch (shp->sh_type) {
1605 case SHT_RELA:
1606 case SHT_REL:
1607 if (shp->sh_addr != 0) {
1608 kobj_free((void *)shp->sh_addr,
1609 shp->sh_size);
1610 }
1611 break;
1612 }
1613 }
1614 kobj_free(omp->shdrs, omp->hdr.e_shentsize * omp->hdr.e_shnum);
1615 }
1616 /*
1617 * Discard the old symbol table and our copy of the module strucure.
1618 */
1619 if (!(mp->flags & KOBJ_PRIM))
1620 kobj_free(omp->symspace, omp->symsize);
1621 kobj_free(omp, sizeof (struct module));
1622 }
1623
1624 static void
kobj_export_ctf(struct module * mp)1625 kobj_export_ctf(struct module *mp)
1626 {
1627 char *data = mp->ctfdata;
1628 size_t size = mp->ctfsize;
1629
1630 if (data != NULL) {
1631 if (_moddebug & MODDEBUG_NOCTF) {
1632 mp->ctfdata = NULL;
1633 mp->ctfsize = 0;
1634 } else {
1635 mp->ctfdata = vmem_alloc(ctf_arena, size,
1636 VM_BESTFIT | VM_SLEEP);
1637 bcopy(data, mp->ctfdata, size);
1638 }
1639
1640 if (!(mp->flags & KOBJ_PRIM))
1641 kobj_free(data, size);
1642 }
1643 }
1644
1645 void
kobj_export_module(struct module * mp)1646 kobj_export_module(struct module *mp)
1647 {
1648 kobj_export_ksyms(mp);
1649 kobj_export_ctf(mp);
1650
1651 mp->flags |= KOBJ_EXPORTED;
1652 }
1653
1654 static int
process_dynamic(struct module * mp,char * dyndata,char * strdata)1655 process_dynamic(struct module *mp, char *dyndata, char *strdata)
1656 {
1657 char *path = NULL, *depstr = NULL;
1658 int allocsize = 0, osize = 0, nsize = 0;
1659 char *libname, *tmp;
1660 int lsize;
1661 Dyn *dynp;
1662
1663 for (dynp = (Dyn *)dyndata; dynp && dynp->d_tag != DT_NULL; dynp++) {
1664 switch (dynp->d_tag) {
1665 case DT_NEEDED:
1666 /*
1667 * Read the DT_NEEDED entries, expanding the macros they
1668 * contain (if any), and concatenating them into a
1669 * single space-separated dependency list.
1670 */
1671 libname = (ulong_t)dynp->d_un.d_ptr + strdata;
1672
1673 if (strchr(libname, '$') != NULL) {
1674 char *_lib;
1675
1676 if (path == NULL)
1677 path = kobj_alloc(MAXPATHLEN, KM_WAIT);
1678 if ((_lib = expand_libmacro(libname, path,
1679 path)) != NULL)
1680 libname = _lib;
1681 else {
1682 _kobj_printf(ops, "krtld: "
1683 "process_dynamic: failed to expand "
1684 "%s\n", libname);
1685 }
1686 }
1687
1688 lsize = strlen(libname);
1689 nsize += lsize;
1690 if (nsize + 1 > allocsize) {
1691 tmp = kobj_alloc(allocsize + MAXPATHLEN,
1692 KM_WAIT);
1693 if (depstr != NULL) {
1694 bcopy(depstr, tmp, osize);
1695 kobj_free(depstr, allocsize);
1696 }
1697 depstr = tmp;
1698 allocsize += MAXPATHLEN;
1699 }
1700 bcopy(libname, depstr + osize, lsize);
1701 *(depstr + nsize) = ' '; /* separator */
1702 nsize++;
1703 osize = nsize;
1704 break;
1705
1706 case DT_FLAGS_1:
1707 if (dynp->d_un.d_val & DF_1_IGNMULDEF)
1708 mp->flags |= KOBJ_IGNMULDEF;
1709 if (dynp->d_un.d_val & DF_1_NOKSYMS)
1710 mp->flags |= KOBJ_NOKSYMS;
1711
1712 break;
1713 }
1714 }
1715
1716 /*
1717 * finish up the depends string (if any)
1718 */
1719 if (depstr != NULL) {
1720 *(depstr + nsize - 1) = '\0'; /* overwrite separator w/term */
1721 if (path != NULL)
1722 kobj_free(path, MAXPATHLEN);
1723
1724 tmp = kobj_alloc(nsize, KM_WAIT);
1725 bcopy(depstr, tmp, nsize);
1726 kobj_free(depstr, allocsize);
1727 depstr = tmp;
1728
1729 mp->depends_on = depstr;
1730 }
1731
1732 return (0);
1733 }
1734
1735 static int
do_dynamic(struct module * mp,struct _buf * file)1736 do_dynamic(struct module *mp, struct _buf *file)
1737 {
1738 Shdr *dshp, *dstrp, *shp;
1739 char *dyndata, *dstrdata;
1740 int dshn, shn, rc;
1741
1742 /* find and validate the dynamic section (if any) */
1743
1744 for (dshp = NULL, shn = 1; shn < mp->hdr.e_shnum; shn++) {
1745 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
1746 switch (shp->sh_type) {
1747 case SHT_DYNAMIC:
1748 if (dshp != NULL) {
1749 _kobj_printf(ops, "krtld: get_dynamic: %s, ",
1750 mp->filename);
1751 _kobj_printf(ops,
1752 "multiple dynamic sections\n");
1753 return (-1);
1754 } else {
1755 dshp = shp;
1756 dshn = shn;
1757 }
1758 break;
1759 }
1760 }
1761
1762 if (dshp == NULL)
1763 return (0);
1764
1765 if (dshp->sh_link > mp->hdr.e_shnum) {
1766 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1767 _kobj_printf(ops, "no section for sh_link %d\n", dshp->sh_link);
1768 return (-1);
1769 }
1770 dstrp = (Shdr *)(mp->shdrs + dshp->sh_link * mp->hdr.e_shentsize);
1771
1772 if (dstrp->sh_type != SHT_STRTAB) {
1773 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1774 _kobj_printf(ops, "sh_link not a string table for section %d\n",
1775 dshn);
1776 return (-1);
1777 }
1778
1779 /* read it from disk */
1780
1781 dyndata = kobj_alloc(dshp->sh_size, KM_WAIT|KM_TMP);
1782 if (kobj_read_file(file, dyndata, dshp->sh_size, dshp->sh_offset) < 0) {
1783 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1784 _kobj_printf(ops, "error reading section %d\n", dshn);
1785
1786 kobj_free(dyndata, dshp->sh_size);
1787 return (-1);
1788 }
1789
1790 dstrdata = kobj_alloc(dstrp->sh_size, KM_WAIT|KM_TMP);
1791 if (kobj_read_file(file, dstrdata, dstrp->sh_size,
1792 dstrp->sh_offset) < 0) {
1793 _kobj_printf(ops, "krtld: get_dynamic: %s, ", mp->filename);
1794 _kobj_printf(ops, "error reading section %d\n", dshp->sh_link);
1795
1796 kobj_free(dyndata, dshp->sh_size);
1797 kobj_free(dstrdata, dstrp->sh_size);
1798 return (-1);
1799 }
1800
1801 /* pull the interesting pieces out */
1802
1803 rc = process_dynamic(mp, dyndata, dstrdata);
1804
1805 kobj_free(dyndata, dshp->sh_size);
1806 kobj_free(dstrdata, dstrp->sh_size);
1807
1808 return (rc);
1809 }
1810
1811 void
kobj_set_ctf(struct module * mp,caddr_t data,size_t size)1812 kobj_set_ctf(struct module *mp, caddr_t data, size_t size)
1813 {
1814 if (!standalone) {
1815 if (mp->ctfdata != NULL) {
1816 if (vmem_contains(ctf_arena, mp->ctfdata,
1817 mp->ctfsize)) {
1818 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
1819 } else {
1820 kobj_free(mp->ctfdata, mp->ctfsize);
1821 }
1822 }
1823 }
1824
1825 /*
1826 * The order is very important here. We need to make sure that
1827 * consumers, at any given instant, see a consistent state. We'd
1828 * rather they see no CTF data than the address of one buffer and the
1829 * size of another.
1830 */
1831 mp->ctfdata = NULL;
1832 membar_producer();
1833 mp->ctfsize = size;
1834 mp->ctfdata = data;
1835 membar_producer();
1836 }
1837
1838 int
kobj_load_module(struct modctl * modp,int use_path)1839 kobj_load_module(struct modctl *modp, int use_path)
1840 {
1841 char *filename = modp->mod_filename;
1842 char *modname = modp->mod_modname;
1843 int i;
1844 int n;
1845 struct _buf *file;
1846 struct module *mp = NULL;
1847 #ifdef MODDIR_SUFFIX
1848 int no_suffixdir_drv = 0;
1849 #endif
1850
1851 mp = kobj_zalloc(sizeof (struct module), KM_WAIT);
1852
1853 /*
1854 * We need to prevent kmdb's symbols from leaking into /dev/ksyms.
1855 * kmdb contains a bunch of symbols with well-known names, symbols
1856 * which will mask the real versions, thus causing no end of trouble
1857 * for mdb.
1858 */
1859 if (strcmp(modp->mod_modname, "kmdbmod") == 0)
1860 mp->flags |= KOBJ_NOKSYMS;
1861
1862 file = kobj_open_path(filename, use_path, 1);
1863 if (file == (struct _buf *)-1) {
1864 #ifdef MODDIR_SUFFIX
1865 file = kobj_open_path(filename, use_path, 0);
1866 #endif
1867 if (file == (struct _buf *)-1) {
1868 kobj_free(mp, sizeof (*mp));
1869 goto bad;
1870 }
1871 #ifdef MODDIR_SUFFIX
1872 /*
1873 * There is no driver module in the ISA specific (suffix)
1874 * subdirectory but there is a module in the parent directory.
1875 */
1876 if (strncmp(filename, "drv/", 4) == 0) {
1877 no_suffixdir_drv = 1;
1878 }
1879 #endif
1880 }
1881
1882 mp->filename = kobj_alloc(strlen(file->_name) + 1, KM_WAIT);
1883 (void) strcpy(mp->filename, file->_name);
1884
1885 if (kobj_read_file(file, (char *)&mp->hdr, sizeof (mp->hdr), 0) < 0) {
1886 _kobj_printf(ops, "kobj_load_module: %s read header failed\n",
1887 modname);
1888 kobj_free(mp->filename, strlen(file->_name) + 1);
1889 kobj_free(mp, sizeof (*mp));
1890 goto bad;
1891 }
1892 for (i = 0; i < SELFMAG; i++) {
1893 if (mp->hdr.e_ident[i] != ELFMAG[i]) {
1894 if (_moddebug & MODDEBUG_ERRMSG)
1895 _kobj_printf(ops, "%s not an elf module\n",
1896 modname);
1897 kobj_free(mp->filename, strlen(file->_name) + 1);
1898 kobj_free(mp, sizeof (*mp));
1899 goto bad;
1900 }
1901 }
1902 /*
1903 * It's ELF, but is it our ISA? Interpreting the header
1904 * from a file for a byte-swapped ISA could cause a huge
1905 * and unsatisfiable value to be passed to kobj_alloc below
1906 * and therefore hang booting.
1907 */
1908 if (!elf_mach_ok(&mp->hdr)) {
1909 if (_moddebug & MODDEBUG_ERRMSG)
1910 _kobj_printf(ops, "%s not an elf module for this ISA\n",
1911 modname);
1912 kobj_free(mp->filename, strlen(file->_name) + 1);
1913 kobj_free(mp, sizeof (*mp));
1914 #ifdef MODDIR_SUFFIX
1915 /*
1916 * The driver mod is not in the ISA specific subdirectory
1917 * and the module in the parent directory is not our ISA.
1918 * If it is our ISA, for now we will silently succeed.
1919 */
1920 if (no_suffixdir_drv == 1) {
1921 cmn_err(CE_CONT, "?NOTICE: %s: 64-bit driver module"
1922 " not found\n", modname);
1923 }
1924 #endif
1925 goto bad;
1926 }
1927
1928 /*
1929 * All modules, save for unix, should be relocatable (as opposed to
1930 * dynamic). Dynamic modules come with PLTs and GOTs, which can't
1931 * currently be processed by krtld.
1932 */
1933 if (mp->hdr.e_type != ET_REL) {
1934 if (_moddebug & MODDEBUG_ERRMSG)
1935 _kobj_printf(ops, "%s isn't a relocatable (ET_REL) "
1936 "module\n", modname);
1937 kobj_free(mp->filename, strlen(file->_name) + 1);
1938 kobj_free(mp, sizeof (*mp));
1939 goto bad;
1940 }
1941
1942 n = mp->hdr.e_shentsize * mp->hdr.e_shnum;
1943 mp->shdrs = kobj_alloc(n, KM_WAIT);
1944
1945 if (kobj_read_file(file, mp->shdrs, n, mp->hdr.e_shoff) < 0) {
1946 _kobj_printf(ops, "kobj_load_module: %s error reading "
1947 "section headers\n", modname);
1948 kobj_free(mp->shdrs, n);
1949 kobj_free(mp->filename, strlen(file->_name) + 1);
1950 kobj_free(mp, sizeof (*mp));
1951 goto bad;
1952 }
1953
1954 kobj_notify(KOBJ_NOTIFY_MODLOADING, modp);
1955 module_assign(modp, mp);
1956
1957 /* read in sections */
1958 if (get_progbits(mp, file) < 0) {
1959 _kobj_printf(ops, "%s error reading sections\n", modname);
1960 goto bad;
1961 }
1962
1963 if (do_dynamic(mp, file) < 0) {
1964 _kobj_printf(ops, "%s error reading dynamic section\n",
1965 modname);
1966 goto bad;
1967 }
1968
1969 modp->mod_text = mp->text;
1970 modp->mod_text_size = mp->text_size;
1971
1972 /* read in symbols; adjust values for each section's real address */
1973 if (get_syms(mp, file) < 0) {
1974 _kobj_printf(ops, "%s error reading symbols\n",
1975 modname);
1976 goto bad;
1977 }
1978
1979 /*
1980 * If we didn't dependency information from the dynamic section, look
1981 * for it the old-fashioned way.
1982 */
1983 if (mp->depends_on == NULL)
1984 mp->depends_on = depends_on(mp);
1985
1986 if (get_ctf(mp, file) < 0) {
1987 _kobj_printf(ops, "%s debug information will not "
1988 "be available\n", modname);
1989 }
1990
1991 /* primary kernel modules do not have a signature section */
1992 if (!(mp->flags & KOBJ_PRIM))
1993 get_signature(mp, file);
1994
1995 #ifdef KOBJ_DEBUG
1996 if (kobj_debug & D_LOADING) {
1997 _kobj_printf(ops, "krtld: file=%s\n", mp->filename);
1998 _kobj_printf(ops, "\ttext:0x%p", mp->text);
1999 _kobj_printf(ops, " size: 0x%lx\n", mp->text_size);
2000 _kobj_printf(ops, "\tdata:0x%p", mp->data);
2001 _kobj_printf(ops, " dsize: 0x%lx\n", mp->data_size);
2002 }
2003 #endif /* KOBJ_DEBUG */
2004
2005 /*
2006 * For primary kernel modules, we defer
2007 * symbol resolution and relocation until
2008 * all primary objects have been loaded.
2009 */
2010 if (!standalone) {
2011 int ddrval, dcrval;
2012 char *dependent_modname;
2013 /* load all dependents */
2014 dependent_modname = kobj_zalloc(MODMAXNAMELEN, KM_WAIT);
2015 ddrval = do_dependents(modp, dependent_modname, MODMAXNAMELEN);
2016
2017 /*
2018 * resolve undefined and common symbols,
2019 * also allocates common space
2020 */
2021 if ((dcrval = do_common(mp)) < 0) {
2022 switch (dcrval) {
2023 case DOSYM_UNSAFE:
2024 _kobj_printf(ops, "WARNING: mod_load: "
2025 "MT-unsafe module '%s' rejected\n",
2026 modname);
2027 break;
2028 case DOSYM_UNDEF:
2029 _kobj_printf(ops, "WARNING: mod_load: "
2030 "cannot load module '%s'\n",
2031 modname);
2032 if (ddrval == -1) {
2033 _kobj_printf(ops, "WARNING: %s: ",
2034 modname);
2035 _kobj_printf(ops,
2036 "unable to resolve dependency, "
2037 "module '%s' not found\n",
2038 dependent_modname);
2039 }
2040 break;
2041 }
2042 }
2043 kobj_free(dependent_modname, MODMAXNAMELEN);
2044 if (dcrval < 0)
2045 goto bad;
2046
2047 /* process relocation tables */
2048 if (do_relocations(mp) < 0) {
2049 _kobj_printf(ops, "%s error doing relocations\n",
2050 modname);
2051 goto bad;
2052 }
2053
2054 if (mp->destination) {
2055 off_t off = (uintptr_t)mp->destination & PAGEOFFSET;
2056 caddr_t base = (caddr_t)mp->destination - off;
2057 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2058
2059 hat_unload(kas.a_hat, base, size, HAT_UNLOAD_UNLOCK);
2060 vmem_free(heap_arena, base, size);
2061 }
2062
2063 /* sync_instruction_memory */
2064 kobj_sync_instruction_memory(mp->text, mp->text_size);
2065 kobj_export_module(mp);
2066 kobj_notify(KOBJ_NOTIFY_MODLOADED, modp);
2067 }
2068 kobj_close_file(file);
2069 return (0);
2070 bad:
2071 if (file != (struct _buf *)-1)
2072 kobj_close_file(file);
2073 if (modp->mod_mp != NULL)
2074 free_module_data(modp->mod_mp);
2075
2076 module_assign(modp, NULL);
2077 return ((file == (struct _buf *)-1) ? ENOENT : EINVAL);
2078 }
2079
2080 int
kobj_load_primary_module(struct modctl * modp)2081 kobj_load_primary_module(struct modctl *modp)
2082 {
2083 struct modctl *dep;
2084 struct module *mp;
2085
2086 if (kobj_load_module(modp, 0) != 0)
2087 return (-1);
2088
2089 dep = NULL;
2090 mp = modp->mod_mp;
2091 mp->flags |= KOBJ_PRIM;
2092
2093 /* Bind new module to its dependents */
2094 if (mp->depends_on != NULL && (dep =
2095 mod_already_loaded(mp->depends_on)) == NULL) {
2096 #ifdef KOBJ_DEBUG
2097 if (kobj_debug & D_DEBUG) {
2098 _kobj_printf(ops, "krtld: failed to resolve deps "
2099 "for primary %s\n", modp->mod_modname);
2100 }
2101 #endif
2102 return (-1);
2103 }
2104
2105 if (dep != NULL)
2106 add_dependent(mp, dep->mod_mp);
2107
2108 /*
2109 * Relocate it. This module may not be part of a link map, so we
2110 * can't use bind_primary.
2111 */
2112 if (do_common(mp) < 0 || do_symbols(mp, 0) < 0 ||
2113 do_relocations(mp) < 0) {
2114 #ifdef KOBJ_DEBUG
2115 if (kobj_debug & D_DEBUG) {
2116 _kobj_printf(ops, "krtld: failed to relocate "
2117 "primary %s\n", modp->mod_modname);
2118 }
2119 #endif
2120 return (-1);
2121 }
2122
2123 return (0);
2124 }
2125
2126 static void
module_assign(struct modctl * cp,struct module * mp)2127 module_assign(struct modctl *cp, struct module *mp)
2128 {
2129 if (standalone) {
2130 cp->mod_mp = mp;
2131 return;
2132 }
2133 mutex_enter(&mod_lock);
2134 cp->mod_mp = mp;
2135 cp->mod_gencount++;
2136 mutex_exit(&mod_lock);
2137 }
2138
2139 void
kobj_unload_module(struct modctl * modp)2140 kobj_unload_module(struct modctl *modp)
2141 {
2142 struct module *mp = modp->mod_mp;
2143
2144 if ((_moddebug & MODDEBUG_KEEPTEXT) && mp) {
2145 _kobj_printf(ops, "text for %s ", mp->filename);
2146 _kobj_printf(ops, "was at %p\n", mp->text);
2147 mp->text = NULL; /* don't actually free it */
2148 }
2149
2150 kobj_notify(KOBJ_NOTIFY_MODUNLOADING, modp);
2151
2152 /*
2153 * Null out mod_mp first, so consumers (debuggers) know not to look
2154 * at the module structure any more.
2155 */
2156 mutex_enter(&mod_lock);
2157 modp->mod_mp = NULL;
2158 mutex_exit(&mod_lock);
2159
2160 kobj_notify(KOBJ_NOTIFY_MODUNLOADED, modp);
2161 free_module_data(mp);
2162 }
2163
2164 static void
free_module_data(struct module * mp)2165 free_module_data(struct module *mp)
2166 {
2167 struct module_list *lp, *tmp;
2168 hotinline_desc_t *hid, *next;
2169 int ksyms_exported = 0;
2170
2171 lp = mp->head;
2172 while (lp) {
2173 tmp = lp;
2174 lp = lp->next;
2175 kobj_free((char *)tmp, sizeof (*tmp));
2176 }
2177
2178 /* release hotinlines */
2179 hid = mp->hi_calls;
2180 while (hid != NULL) {
2181 next = hid->hid_next;
2182 kobj_free(hid->hid_symname, strlen(hid->hid_symname) + 1);
2183 kobj_free(hid, sizeof (hotinline_desc_t));
2184 hid = next;
2185 }
2186
2187 rw_enter(&ksyms_lock, RW_WRITER);
2188 if (mp->symspace) {
2189 if (vmem_contains(ksyms_arena, mp->symspace, mp->symsize)) {
2190 vmem_free(ksyms_arena, mp->symspace, mp->symsize);
2191 ksyms_exported = 1;
2192 } else {
2193 if (mp->flags & KOBJ_NOKSYMS)
2194 ksyms_exported = 1;
2195 kobj_free(mp->symspace, mp->symsize);
2196 }
2197 }
2198 rw_exit(&ksyms_lock);
2199
2200 if (mp->ctfdata) {
2201 if (vmem_contains(ctf_arena, mp->ctfdata, mp->ctfsize))
2202 vmem_free(ctf_arena, mp->ctfdata, mp->ctfsize);
2203 else
2204 kobj_free(mp->ctfdata, mp->ctfsize);
2205 }
2206
2207 if (mp->sigdata)
2208 kobj_free(mp->sigdata, mp->sigsize);
2209
2210 /*
2211 * We did not get far enough into kobj_export_ksyms() to free allocated
2212 * buffers because we encounted error conditions. Free the buffers.
2213 */
2214 if ((ksyms_exported == 0) && (mp->shdrs != NULL)) {
2215 uint_t shn;
2216 Shdr *shp;
2217
2218 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2219 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2220 switch (shp->sh_type) {
2221 case SHT_RELA:
2222 case SHT_REL:
2223 if (shp->sh_addr != 0)
2224 kobj_free((void *)shp->sh_addr,
2225 shp->sh_size);
2226 break;
2227 }
2228 }
2229
2230 if (!(mp->flags & KOBJ_PRIM)) {
2231 kobj_free(mp->shdrs,
2232 mp->hdr.e_shentsize * mp->hdr.e_shnum);
2233 }
2234 }
2235
2236 if (mp->bss)
2237 vmem_free(data_arena, (void *)mp->bss, mp->bss_size);
2238
2239 if (mp->fbt_tab)
2240 kobj_texthole_free(mp->fbt_tab, mp->fbt_size);
2241
2242 if (mp->textwin_base)
2243 kobj_textwin_free(mp);
2244
2245 if (mp->sdt_probes != NULL) {
2246 sdt_probedesc_t *sdp = mp->sdt_probes, *next;
2247
2248 while (sdp != NULL) {
2249 next = sdp->sdpd_next;
2250 kobj_free(sdp->sdpd_name, strlen(sdp->sdpd_name) + 1);
2251 kobj_free(sdp, sizeof (sdt_probedesc_t));
2252 sdp = next;
2253 }
2254 }
2255
2256 if (mp->sdt_tab)
2257 kobj_texthole_free(mp->sdt_tab, mp->sdt_size);
2258 if (mp->text)
2259 vmem_free(text_arena, mp->text, mp->text_size);
2260 if (mp->data)
2261 vmem_free(data_arena, mp->data, mp->data_size);
2262 if (mp->depends_on)
2263 kobj_free(mp->depends_on, strlen(mp->depends_on)+1);
2264 if (mp->filename)
2265 kobj_free(mp->filename, strlen(mp->filename)+1);
2266
2267 kobj_free((char *)mp, sizeof (*mp));
2268 }
2269
2270 static int
get_progbits(struct module * mp,struct _buf * file)2271 get_progbits(struct module *mp, struct _buf *file)
2272 {
2273 struct proginfo *tp, *dp, *sdp;
2274 Shdr *shp;
2275 reloc_dest_t dest = NULL;
2276 uintptr_t bits_ptr;
2277 uintptr_t text = 0, data, textptr;
2278 uint_t shn;
2279 int err = -1;
2280
2281 tp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2282 dp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2283 sdp = kobj_zalloc(sizeof (struct proginfo), KM_WAIT|KM_TMP);
2284 /*
2285 * loop through sections to find out how much space we need
2286 * for text, data, (also bss that is already assigned)
2287 */
2288 if (get_progbits_size(mp, tp, dp, sdp) < 0)
2289 goto done;
2290
2291 mp->text_size = tp->size;
2292 mp->data_size = dp->size;
2293
2294 if (standalone) {
2295 caddr_t limit = _data;
2296
2297 if (lg_pagesize && _text + lg_pagesize < limit)
2298 limit = _text + lg_pagesize;
2299
2300 mp->text = kobj_segbrk(&_etext, mp->text_size,
2301 tp->align, limit);
2302 /*
2303 * If we can't grow the text segment, try the
2304 * data segment before failing.
2305 */
2306 if (mp->text == NULL) {
2307 mp->text = kobj_segbrk(&_edata, mp->text_size,
2308 tp->align, 0);
2309 }
2310
2311 mp->data = kobj_segbrk(&_edata, mp->data_size, dp->align, 0);
2312
2313 if (mp->text == NULL || mp->data == NULL)
2314 goto done;
2315
2316 } else {
2317 if (text_arena == NULL)
2318 kobj_vmem_init(&text_arena, &data_arena);
2319
2320 /*
2321 * some architectures may want to load the module on a
2322 * page that is currently read only. It may not be
2323 * possible for those architectures to remap their page
2324 * on the fly. So we provide a facility for them to hang
2325 * a private hook where the memory they assign the module
2326 * is not the actual place where the module loads.
2327 *
2328 * In this case there are two addresses that deal with the
2329 * modload.
2330 * 1) the final destination of the module
2331 * 2) the address that is used to view the newly
2332 * loaded module until all the relocations relative to 1
2333 * above are completed.
2334 *
2335 * That is what dest is used for below.
2336 */
2337 mp->text_size += tp->align;
2338 mp->data_size += dp->align;
2339
2340 mp->text = kobj_text_alloc(text_arena, mp->text_size);
2341
2342 /*
2343 * a remap is taking place. Align the text ptr relative
2344 * to the secondary mapping. That is where the bits will
2345 * be read in.
2346 */
2347 if (kvseg.s_base != NULL && !vmem_contains(heaptext_arena,
2348 mp->text, mp->text_size)) {
2349 off_t off = (uintptr_t)mp->text & PAGEOFFSET;
2350 size_t size = P2ROUNDUP(mp->text_size + off, PAGESIZE);
2351 caddr_t map = vmem_alloc(heap_arena, size, VM_SLEEP);
2352 caddr_t orig = mp->text - off;
2353 pgcnt_t pages = size / PAGESIZE;
2354
2355 dest = (reloc_dest_t)(map + off);
2356 text = ALIGN((uintptr_t)dest, tp->align);
2357
2358 while (pages--) {
2359 hat_devload(kas.a_hat, map, PAGESIZE,
2360 hat_getpfnum(kas.a_hat, orig),
2361 PROT_READ | PROT_WRITE | PROT_EXEC,
2362 HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
2363 map += PAGESIZE;
2364 orig += PAGESIZE;
2365 }
2366 /*
2367 * Since we set up a non-cacheable mapping, we need
2368 * to flush any old entries in the cache that might
2369 * be left around from the read-only mapping.
2370 */
2371 dcache_flushall();
2372 }
2373 if (mp->data_size)
2374 mp->data = vmem_alloc(data_arena, mp->data_size,
2375 VM_SLEEP | VM_BESTFIT);
2376 }
2377 textptr = (uintptr_t)mp->text;
2378 textptr = ALIGN(textptr, tp->align);
2379 mp->destination = dest;
2380
2381 /*
2382 * This is the case where a remap is not being done.
2383 */
2384 if (text == 0)
2385 text = ALIGN((uintptr_t)mp->text, tp->align);
2386 data = ALIGN((uintptr_t)mp->data, dp->align);
2387
2388 /* now loop though sections assigning addresses and loading the data */
2389 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2390 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2391 if (!(shp->sh_flags & SHF_ALLOC))
2392 continue;
2393
2394 if ((shp->sh_flags & SHF_WRITE) == 0)
2395 bits_ptr = text;
2396 else
2397 bits_ptr = data;
2398
2399 bits_ptr = ALIGN(bits_ptr, shp->sh_addralign);
2400
2401 if (shp->sh_type == SHT_NOBITS) {
2402 /*
2403 * Zero bss.
2404 */
2405 bzero((caddr_t)bits_ptr, shp->sh_size);
2406 shp->sh_type = SHT_PROGBITS;
2407 } else {
2408 if (kobj_read_file(file, (char *)bits_ptr,
2409 shp->sh_size, shp->sh_offset) < 0)
2410 goto done;
2411 }
2412
2413 if (shp->sh_flags & SHF_WRITE) {
2414 shp->sh_addr = bits_ptr;
2415 } else {
2416 textptr = ALIGN(textptr, shp->sh_addralign);
2417 shp->sh_addr = textptr;
2418 textptr += shp->sh_size;
2419 }
2420
2421 bits_ptr += shp->sh_size;
2422 if ((shp->sh_flags & SHF_WRITE) == 0)
2423 text = bits_ptr;
2424 else
2425 data = bits_ptr;
2426 }
2427
2428 err = 0;
2429 done:
2430 /*
2431 * Free and mark as freed the section headers here so that
2432 * free_module_data() does not have to worry about this buffer.
2433 *
2434 * This buffer is freed here because one of the possible reasons
2435 * for error is a section with non-zero sh_addr and in that case
2436 * free_module_data() would have no way of recognizing that this
2437 * buffer was unallocated.
2438 */
2439 if (err != 0) {
2440 kobj_free(mp->shdrs, mp->hdr.e_shentsize * mp->hdr.e_shnum);
2441 mp->shdrs = NULL;
2442 }
2443
2444 (void) kobj_free(tp, sizeof (struct proginfo));
2445 (void) kobj_free(dp, sizeof (struct proginfo));
2446 (void) kobj_free(sdp, sizeof (struct proginfo));
2447
2448 return (err);
2449 }
2450
2451 /*
2452 * Go through suppress_sym_list to see if "multiply defined"
2453 * warning of this symbol should be suppressed. Return 1 if
2454 * warning should be suppressed, 0 otherwise.
2455 */
2456 static int
kobj_suppress_warning(char * symname)2457 kobj_suppress_warning(char *symname)
2458 {
2459 int i;
2460
2461 for (i = 0; suppress_sym_list[i] != NULL; i++) {
2462 if (strcmp(suppress_sym_list[i], symname) == 0)
2463 return (1);
2464 }
2465
2466 return (0);
2467 }
2468
2469 static int
get_syms(struct module * mp,struct _buf * file)2470 get_syms(struct module *mp, struct _buf *file)
2471 {
2472 uint_t shn;
2473 Shdr *shp;
2474 uint_t i;
2475 Sym *sp, *ksp;
2476 char *symname;
2477 int dosymtab = 0;
2478
2479 /*
2480 * Find the interesting sections.
2481 */
2482 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2483 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2484 switch (shp->sh_type) {
2485 case SHT_SYMTAB:
2486 mp->symtbl_section = shn;
2487 mp->symhdr = shp;
2488 dosymtab++;
2489 break;
2490
2491 case SHT_RELA:
2492 case SHT_REL:
2493 /*
2494 * Already loaded.
2495 */
2496 if (shp->sh_addr)
2497 continue;
2498
2499 /* KM_TMP since kobj_free'd in do_relocations */
2500 shp->sh_addr = (Addr)
2501 kobj_alloc(shp->sh_size, KM_WAIT|KM_TMP);
2502
2503 if (kobj_read_file(file, (char *)shp->sh_addr,
2504 shp->sh_size, shp->sh_offset) < 0) {
2505 _kobj_printf(ops, "krtld: get_syms: %s, ",
2506 mp->filename);
2507 _kobj_printf(ops, "error reading section %d\n",
2508 shn);
2509 return (-1);
2510 }
2511 break;
2512 }
2513 }
2514
2515 /*
2516 * This is true for a stripped executable. In the case of
2517 * 'unix' it can be stripped but it still contains the SHT_DYNSYM,
2518 * and since that symbol information is still present everything
2519 * is just fine.
2520 */
2521 if (!dosymtab) {
2522 if (mp->flags & KOBJ_EXEC)
2523 return (0);
2524 _kobj_printf(ops, "krtld: get_syms: %s ",
2525 mp->filename);
2526 _kobj_printf(ops, "no SHT_SYMTAB symbol table found\n");
2527 return (-1);
2528 }
2529
2530 /*
2531 * get the associated string table header
2532 */
2533 if ((mp->symhdr == 0) || (mp->symhdr->sh_link >= mp->hdr.e_shnum))
2534 return (-1);
2535 mp->strhdr = (Shdr *)
2536 (mp->shdrs + mp->symhdr->sh_link * mp->hdr.e_shentsize);
2537
2538 mp->nsyms = mp->symhdr->sh_size / mp->symhdr->sh_entsize;
2539 mp->hashsize = kobj_gethashsize(mp->nsyms);
2540
2541 /*
2542 * Allocate space for the symbol table, buckets, chains, and strings.
2543 */
2544 mp->symsize = mp->symhdr->sh_size +
2545 (mp->hashsize + mp->nsyms) * sizeof (symid_t) + mp->strhdr->sh_size;
2546 mp->symspace = kobj_zalloc(mp->symsize, KM_WAIT|KM_SCRATCH);
2547
2548 mp->symtbl = mp->symspace;
2549 mp->buckets = (symid_t *)(mp->symtbl + mp->symhdr->sh_size);
2550 mp->chains = mp->buckets + mp->hashsize;
2551 mp->strings = (char *)(mp->chains + mp->nsyms);
2552
2553 if (kobj_read_file(file, mp->symtbl,
2554 mp->symhdr->sh_size, mp->symhdr->sh_offset) < 0 ||
2555 kobj_read_file(file, mp->strings,
2556 mp->strhdr->sh_size, mp->strhdr->sh_offset) < 0)
2557 return (-1);
2558
2559 /*
2560 * loop through the symbol table adjusting values to account
2561 * for where each section got loaded into memory. Also
2562 * fill in the hash table.
2563 */
2564 for (i = 1; i < mp->nsyms; i++) {
2565 sp = (Sym *)(mp->symtbl + i * mp->symhdr->sh_entsize);
2566 if (sp->st_shndx < SHN_LORESERVE) {
2567 if (sp->st_shndx >= mp->hdr.e_shnum) {
2568 _kobj_printf(ops, "%s bad shndx ",
2569 file->_name);
2570 _kobj_printf(ops, "in symbol %d\n", i);
2571 return (-1);
2572 }
2573 shp = (Shdr *)
2574 (mp->shdrs +
2575 sp->st_shndx * mp->hdr.e_shentsize);
2576 if (!(mp->flags & KOBJ_EXEC))
2577 sp->st_value += shp->sh_addr;
2578 }
2579
2580 if (sp->st_name == 0 || sp->st_shndx == SHN_UNDEF)
2581 continue;
2582 if (sp->st_name >= mp->strhdr->sh_size)
2583 return (-1);
2584
2585 symname = mp->strings + sp->st_name;
2586
2587 if (!(mp->flags & KOBJ_EXEC) &&
2588 ELF_ST_BIND(sp->st_info) == STB_GLOBAL) {
2589 ksp = kobj_lookup_all(mp, symname, 0);
2590
2591 if (ksp && ELF_ST_BIND(ksp->st_info) == STB_GLOBAL &&
2592 !kobj_suppress_warning(symname) &&
2593 sp->st_shndx != SHN_UNDEF &&
2594 sp->st_shndx != SHN_COMMON &&
2595 ksp->st_shndx != SHN_UNDEF &&
2596 ksp->st_shndx != SHN_COMMON) {
2597 /*
2598 * Unless this symbol is a stub, it's multiply
2599 * defined. Multiply-defined symbols are
2600 * usually bad, but some objects (kmdb) have
2601 * a legitimate need to have their own
2602 * copies of common functions.
2603 */
2604 if ((standalone ||
2605 ksp->st_value < (uintptr_t)stubs_base ||
2606 ksp->st_value >= (uintptr_t)stubs_end) &&
2607 !(mp->flags & KOBJ_IGNMULDEF)) {
2608 _kobj_printf(ops,
2609 "%s symbol ", file->_name);
2610 _kobj_printf(ops,
2611 "%s multiply defined\n", symname);
2612 }
2613 }
2614 }
2615
2616 sym_insert(mp, symname, i);
2617 }
2618
2619 return (0);
2620 }
2621
2622 static int
get_ctf(struct module * mp,struct _buf * file)2623 get_ctf(struct module *mp, struct _buf *file)
2624 {
2625 char *shstrtab, *ctfdata;
2626 size_t shstrlen;
2627 Shdr *shp;
2628 uint_t i;
2629
2630 if (_moddebug & MODDEBUG_NOCTF)
2631 return (0); /* do not attempt to even load CTF data */
2632
2633 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2634 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2635 mp->filename);
2636 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2637 mp->hdr.e_shstrndx);
2638 return (-1);
2639 }
2640
2641 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2642 shstrlen = shp->sh_size;
2643 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2644
2645 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2646 _kobj_printf(ops, "krtld: get_ctf: %s, ",
2647 mp->filename);
2648 _kobj_printf(ops, "error reading section %u\n",
2649 mp->hdr.e_shstrndx);
2650 kobj_free(shstrtab, shstrlen);
2651 return (-1);
2652 }
2653
2654 for (i = 0; i < mp->hdr.e_shnum; i++) {
2655 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2656
2657 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2658 strcmp(shstrtab + shp->sh_name, ".SUNW_ctf") == 0) {
2659 ctfdata = kobj_alloc(shp->sh_size, KM_WAIT|KM_SCRATCH);
2660
2661 if (kobj_read_file(file, ctfdata, shp->sh_size,
2662 shp->sh_offset) < 0) {
2663 _kobj_printf(ops, "krtld: get_ctf: %s, error "
2664 "reading .SUNW_ctf data\n", mp->filename);
2665 kobj_free(ctfdata, shp->sh_size);
2666 kobj_free(shstrtab, shstrlen);
2667 return (-1);
2668 }
2669
2670 mp->ctfdata = ctfdata;
2671 mp->ctfsize = shp->sh_size;
2672 break;
2673 }
2674 }
2675
2676 kobj_free(shstrtab, shstrlen);
2677 return (0);
2678 }
2679
2680 #define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */
2681
2682 /*
2683 * Return the hash of the ELF sections that are memory resident.
2684 * i.e. text and data. We skip a SHT_NOBITS section since it occupies
2685 * no space in the file. We use SHA1 here since libelfsign uses
2686 * it and both places need to use the same algorithm.
2687 */
2688 static void
crypto_es_hash(struct module * mp,char * hash,char * shstrtab)2689 crypto_es_hash(struct module *mp, char *hash, char *shstrtab)
2690 {
2691 uint_t shn;
2692 Shdr *shp;
2693 SHA1_CTX ctx;
2694
2695 SHA1Init(&ctx);
2696
2697 for (shn = 1; shn < mp->hdr.e_shnum; shn++) {
2698 shp = (Shdr *)(mp->shdrs + shn * mp->hdr.e_shentsize);
2699 if (!(shp->sh_flags & SHF_ALLOC) || shp->sh_size == 0)
2700 continue;
2701
2702 /*
2703 * The check should ideally be shp->sh_type == SHT_NOBITS.
2704 * However, we can't do that check here as get_progbits()
2705 * resets the type.
2706 */
2707 if (strcmp(shstrtab + shp->sh_name, ".bss") == 0)
2708 continue;
2709 #ifdef KOBJ_DEBUG
2710 if (kobj_debug & D_DEBUG)
2711 _kobj_printf(ops,
2712 "krtld: crypto_es_hash: updating hash with"
2713 " %s data size=%lx\n", shstrtab + shp->sh_name,
2714 (size_t)shp->sh_size);
2715 #endif
2716 ASSERT(shp->sh_addr != 0);
2717 SHA1Update(&ctx, (const uint8_t *)shp->sh_addr, shp->sh_size);
2718 }
2719
2720 SHA1Final((uchar_t *)hash, &ctx);
2721 }
2722
2723 /*
2724 * Get the .SUNW_signature section for the module, it it exists.
2725 *
2726 * This section exists only for crypto modules. None of the
2727 * primary modules have this section currently.
2728 */
2729 static void
get_signature(struct module * mp,struct _buf * file)2730 get_signature(struct module *mp, struct _buf *file)
2731 {
2732 char *shstrtab, *sigdata = NULL;
2733 size_t shstrlen;
2734 Shdr *shp;
2735 uint_t i;
2736
2737 if (mp->hdr.e_shstrndx >= mp->hdr.e_shnum) {
2738 _kobj_printf(ops, "krtld: get_signature: %s, ",
2739 mp->filename);
2740 _kobj_printf(ops, "corrupt e_shstrndx %u\n",
2741 mp->hdr.e_shstrndx);
2742 return;
2743 }
2744
2745 shp = (Shdr *)(mp->shdrs + mp->hdr.e_shstrndx * mp->hdr.e_shentsize);
2746 shstrlen = shp->sh_size;
2747 shstrtab = kobj_alloc(shstrlen, KM_WAIT|KM_TMP);
2748
2749 if (kobj_read_file(file, shstrtab, shstrlen, shp->sh_offset) < 0) {
2750 _kobj_printf(ops, "krtld: get_signature: %s, ",
2751 mp->filename);
2752 _kobj_printf(ops, "error reading section %u\n",
2753 mp->hdr.e_shstrndx);
2754 kobj_free(shstrtab, shstrlen);
2755 return;
2756 }
2757
2758 for (i = 0; i < mp->hdr.e_shnum; i++) {
2759 shp = (Shdr *)(mp->shdrs + i * mp->hdr.e_shentsize);
2760 if (shp->sh_size != 0 && shp->sh_name < shstrlen &&
2761 strcmp(shstrtab + shp->sh_name,
2762 ELF_SIGNATURE_SECTION) == 0) {
2763 filesig_vers_t filesig_version;
2764 size_t sigsize = shp->sh_size + SHA1_DIGEST_LENGTH;
2765 sigdata = kobj_alloc(sigsize, KM_WAIT|KM_SCRATCH);
2766
2767 if (kobj_read_file(file, sigdata, shp->sh_size,
2768 shp->sh_offset) < 0) {
2769 _kobj_printf(ops, "krtld: get_signature: %s,"
2770 " error reading .SUNW_signature data\n",
2771 mp->filename);
2772 kobj_free(sigdata, sigsize);
2773 kobj_free(shstrtab, shstrlen);
2774 return;
2775 }
2776 filesig_version = ((struct filesignatures *)sigdata)->
2777 filesig_sig.filesig_version;
2778 if (!(filesig_version == FILESIG_VERSION1 ||
2779 filesig_version == FILESIG_VERSION3)) {
2780 /* skip versions we don't understand */
2781 kobj_free(sigdata, sigsize);
2782 kobj_free(shstrtab, shstrlen);
2783 return;
2784 }
2785
2786 mp->sigdata = sigdata;
2787 mp->sigsize = sigsize;
2788 break;
2789 }
2790 }
2791
2792 if (sigdata != NULL) {
2793 crypto_es_hash(mp, sigdata + shp->sh_size, shstrtab);
2794 }
2795
2796 kobj_free(shstrtab, shstrlen);
2797 }
2798
2799 static void
add_dependent(struct module * mp,struct module * dep)2800 add_dependent(struct module *mp, struct module *dep)
2801 {
2802 struct module_list *lp;
2803
2804 for (lp = mp->head; lp; lp = lp->next) {
2805 if (lp->mp == dep)
2806 return; /* already on the list */
2807 }
2808
2809 if (lp == NULL) {
2810 lp = kobj_zalloc(sizeof (*lp), KM_WAIT);
2811
2812 lp->mp = dep;
2813 lp->next = NULL;
2814 if (mp->tail)
2815 mp->tail->next = lp;
2816 else
2817 mp->head = lp;
2818 mp->tail = lp;
2819 }
2820 }
2821
2822 static int
do_dependents(struct modctl * modp,char * modname,size_t modnamelen)2823 do_dependents(struct modctl *modp, char *modname, size_t modnamelen)
2824 {
2825 struct module *mp;
2826 struct modctl *req;
2827 char *d, *p, *q;
2828 int c;
2829 char *err_modname = NULL;
2830
2831 mp = modp->mod_mp;
2832
2833 if ((p = mp->depends_on) == NULL)
2834 return (0);
2835
2836 for (;;) {
2837 /*
2838 * Skip space.
2839 */
2840 while (*p && (*p == ' ' || *p == '\t'))
2841 p++;
2842 /*
2843 * Get module name.
2844 */
2845 d = p;
2846 q = modname;
2847 c = 0;
2848 while (*p && *p != ' ' && *p != '\t') {
2849 if (c < modnamelen - 1) {
2850 *q++ = *p;
2851 c++;
2852 }
2853 p++;
2854 }
2855
2856 if (q == modname)
2857 break;
2858
2859 if (c == modnamelen - 1) {
2860 char *dep = kobj_alloc(p - d + 1, KM_WAIT|KM_TMP);
2861
2862 (void) strncpy(dep, d, p - d + 1);
2863 dep[p - d] = '\0';
2864
2865 _kobj_printf(ops, "%s: dependency ", modp->mod_modname);
2866 _kobj_printf(ops, "'%s' too long ", dep);
2867 _kobj_printf(ops, "(max %d chars)\n", (int)modnamelen);
2868
2869 kobj_free(dep, p - d + 1);
2870
2871 return (-1);
2872 }
2873
2874 *q = '\0';
2875 if ((req = mod_load_requisite(modp, modname)) == NULL) {
2876 #ifndef KOBJ_DEBUG
2877 if (_moddebug & MODDEBUG_LOADMSG) {
2878 #endif /* KOBJ_DEBUG */
2879 _kobj_printf(ops,
2880 "%s: unable to resolve dependency, ",
2881 modp->mod_modname);
2882 _kobj_printf(ops, "cannot load module '%s'\n",
2883 modname);
2884 #ifndef KOBJ_DEBUG
2885 }
2886 #endif /* KOBJ_DEBUG */
2887 if (err_modname == NULL) {
2888 /*
2889 * This must be the same size as the modname
2890 * one.
2891 */
2892 err_modname = kobj_zalloc(MODMAXNAMELEN,
2893 KM_WAIT);
2894
2895 /*
2896 * We can use strcpy() here without fearing
2897 * the NULL terminator because the size of
2898 * err_modname is the same as one of modname,
2899 * and it's filled with zeros.
2900 */
2901 (void) strcpy(err_modname, modname);
2902 }
2903 continue;
2904 }
2905
2906 add_dependent(mp, req->mod_mp);
2907 mod_release_mod(req);
2908
2909 }
2910
2911 if (err_modname != NULL) {
2912 /*
2913 * Copy the first module name where you detect an error to keep
2914 * its behavior the same as before.
2915 * This way keeps minimizing the memory use for error
2916 * modules, and this might be important at boot time because
2917 * the memory usage is a crucial factor for booting in most
2918 * cases. You can expect more verbose messages when using
2919 * a debug kernel or setting a bit in moddebug.
2920 */
2921 bzero(modname, MODMAXNAMELEN);
2922 (void) strcpy(modname, err_modname);
2923 kobj_free(err_modname, MODMAXNAMELEN);
2924 return (-1);
2925 }
2926
2927 return (0);
2928 }
2929
2930 static int
do_common(struct module * mp)2931 do_common(struct module *mp)
2932 {
2933 int err;
2934
2935 /*
2936 * first time through, assign all symbols defined in other
2937 * modules, and count up how much common space will be needed
2938 * (bss_size and bss_align)
2939 */
2940 if ((err = do_symbols(mp, 0)) < 0)
2941 return (err);
2942 /*
2943 * increase bss_size by the maximum delta that could be
2944 * computed by the ALIGN below
2945 */
2946 mp->bss_size += mp->bss_align;
2947 if (mp->bss_size) {
2948 if (standalone)
2949 mp->bss = (uintptr_t)kobj_segbrk(&_edata, mp->bss_size,
2950 MINALIGN, 0);
2951 else
2952 mp->bss = (uintptr_t)vmem_alloc(data_arena,
2953 mp->bss_size, VM_SLEEP | VM_BESTFIT);
2954 bzero((void *)mp->bss, mp->bss_size);
2955 /* now assign addresses to all common symbols */
2956 if ((err = do_symbols(mp, ALIGN(mp->bss, mp->bss_align))) < 0)
2957 return (err);
2958 }
2959 return (0);
2960 }
2961
2962 static int
do_symbols(struct module * mp,Elf64_Addr bss_base)2963 do_symbols(struct module *mp, Elf64_Addr bss_base)
2964 {
2965 int bss_align;
2966 uintptr_t bss_ptr;
2967 int err;
2968 int i;
2969 Sym *sp, *sp1;
2970 char *name;
2971 int assign;
2972 int resolved = 1;
2973
2974 /*
2975 * Nothing left to do (optimization).
2976 */
2977 if (mp->flags & KOBJ_RESOLVED)
2978 return (0);
2979
2980 assign = (bss_base) ? 1 : 0;
2981 bss_ptr = bss_base;
2982 bss_align = 0;
2983 err = 0;
2984
2985 for (i = 1; i < mp->nsyms; i++) {
2986 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * i);
2987 /*
2988 * we know that st_name is in bounds, since get_sections
2989 * has already checked all of the symbols
2990 */
2991 name = mp->strings + sp->st_name;
2992 if (sp->st_shndx != SHN_UNDEF && sp->st_shndx != SHN_COMMON)
2993 continue;
2994 #if defined(__sparc)
2995 /*
2996 * Register symbols are ignored in the kernel
2997 */
2998 if (ELF_ST_TYPE(sp->st_info) == STT_SPARC_REGISTER) {
2999 if (*name != '\0') {
3000 _kobj_printf(ops, "%s: named REGISTER symbol ",
3001 mp->filename);
3002 _kobj_printf(ops, "not supported '%s'\n",
3003 name);
3004 err = DOSYM_UNDEF;
3005 }
3006 continue;
3007 }
3008 #endif /* __sparc */
3009 /*
3010 * TLS symbols are ignored in the kernel
3011 */
3012 if (ELF_ST_TYPE(sp->st_info) == STT_TLS) {
3013 _kobj_printf(ops, "%s: TLS symbol ",
3014 mp->filename);
3015 _kobj_printf(ops, "not supported '%s'\n",
3016 name);
3017 err = DOSYM_UNDEF;
3018 continue;
3019 }
3020
3021 if (ELF_ST_BIND(sp->st_info) != STB_LOCAL) {
3022 if ((sp1 = kobj_lookup_all(mp, name, 0)) != NULL) {
3023 sp->st_shndx = SHN_ABS;
3024 sp->st_value = sp1->st_value;
3025 continue;
3026 }
3027 }
3028
3029 if (sp->st_shndx == SHN_UNDEF) {
3030 resolved = 0;
3031
3032 /*
3033 * Skip over sdt probes and smap calls,
3034 * they're relocated later.
3035 */
3036 if (strncmp(name, sdt_prefix, strlen(sdt_prefix)) == 0)
3037 continue;
3038 #if defined(__x86)
3039 if (strcmp(name, "smap_enable") == 0 ||
3040 strcmp(name, "smap_disable") == 0)
3041 continue;
3042 #endif /* defined(__x86) */
3043
3044
3045 /*
3046 * If it's not a weak reference and it's
3047 * not a primary object, it's an error.
3048 * (Primary objects may take more than
3049 * one pass to resolve)
3050 */
3051 if (!(mp->flags & KOBJ_PRIM) &&
3052 ELF_ST_BIND(sp->st_info) != STB_WEAK) {
3053 _kobj_printf(ops, "%s: undefined symbol",
3054 mp->filename);
3055 _kobj_printf(ops, " '%s'\n", name);
3056 /*
3057 * Try to determine whether this symbol
3058 * represents a dependency on obsolete
3059 * unsafe driver support. This is just
3060 * to make the warning more informative.
3061 */
3062 if (strcmp(name, "sleep") == 0 ||
3063 strcmp(name, "unsleep") == 0 ||
3064 strcmp(name, "wakeup") == 0 ||
3065 strcmp(name, "bsd_compat_ioctl") == 0 ||
3066 strcmp(name, "unsafe_driver") == 0 ||
3067 strncmp(name, "spl", 3) == 0 ||
3068 strncmp(name, "i_ddi_spl", 9) == 0)
3069 err = DOSYM_UNSAFE;
3070 if (err == 0)
3071 err = DOSYM_UNDEF;
3072 }
3073 continue;
3074 }
3075 /*
3076 * It's a common symbol - st_value is the
3077 * required alignment.
3078 */
3079 if (sp->st_value > bss_align)
3080 bss_align = sp->st_value;
3081 bss_ptr = ALIGN(bss_ptr, sp->st_value);
3082 if (assign) {
3083 sp->st_shndx = SHN_ABS;
3084 sp->st_value = bss_ptr;
3085 }
3086 bss_ptr += sp->st_size;
3087 }
3088 if (err)
3089 return (err);
3090 if (assign == 0 && mp->bss == 0) {
3091 mp->bss_align = bss_align;
3092 mp->bss_size = bss_ptr;
3093 } else if (resolved) {
3094 mp->flags |= KOBJ_RESOLVED;
3095 }
3096
3097 return (0);
3098 }
3099
3100 uint_t
kobj_hash_name(const char * p)3101 kobj_hash_name(const char *p)
3102 {
3103 uint_t g;
3104 uint_t hval;
3105
3106 hval = 0;
3107 while (*p) {
3108 hval = (hval << 4) + *p++;
3109 if ((g = (hval & 0xf0000000)) != 0)
3110 hval ^= g >> 24;
3111 hval &= ~g;
3112 }
3113 return (hval);
3114 }
3115
3116 /* look for name in all modules */
3117 uintptr_t
kobj_getsymvalue(char * name,int kernelonly)3118 kobj_getsymvalue(char *name, int kernelonly)
3119 {
3120 Sym *sp;
3121 struct modctl *modp;
3122 struct module *mp;
3123 uintptr_t value = 0;
3124
3125 if ((sp = kobj_lookup_kernel(name)) != NULL)
3126 return ((uintptr_t)sp->st_value);
3127
3128 if (kernelonly)
3129 return (0); /* didn't find it in the kernel so give up */
3130
3131 mutex_enter(&mod_lock);
3132 modp = &modules;
3133 do {
3134 mp = (struct module *)modp->mod_mp;
3135 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3136 (sp = lookup_one(mp, name))) {
3137 value = (uintptr_t)sp->st_value;
3138 break;
3139 }
3140 } while ((modp = modp->mod_next) != &modules);
3141 mutex_exit(&mod_lock);
3142 return (value);
3143 }
3144
3145 /* look for a symbol near value. */
3146 char *
kobj_getsymname(uintptr_t value,ulong_t * offset)3147 kobj_getsymname(uintptr_t value, ulong_t *offset)
3148 {
3149 char *name = NULL;
3150 struct modctl *modp;
3151
3152 struct modctl_list *lp;
3153 struct module *mp;
3154
3155 /*
3156 * Trap handler got us there, but we may not have whole kernel yet.
3157 */
3158 if (standalone)
3159 return (NULL);
3160
3161 /*
3162 * Loop through the primary kernel modules.
3163 */
3164 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3165 mp = mod(lp);
3166
3167 if ((name = kobj_searchsym(mp, value, offset)) != NULL)
3168 return (name);
3169 }
3170
3171 mutex_enter(&mod_lock);
3172 modp = &modules;
3173 do {
3174 mp = (struct module *)modp->mod_mp;
3175 if (mp && !(mp->flags & KOBJ_PRIM) && modp->mod_loaded &&
3176 (name = kobj_searchsym(mp, value, offset)))
3177 break;
3178 } while ((modp = modp->mod_next) != &modules);
3179 mutex_exit(&mod_lock);
3180 return (name);
3181 }
3182
3183 /* return address of symbol and size */
3184
3185 uintptr_t
kobj_getelfsym(char * name,void * mp,int * size)3186 kobj_getelfsym(char *name, void *mp, int *size)
3187 {
3188 Sym *sp;
3189
3190 if (mp == NULL)
3191 sp = kobj_lookup_kernel(name);
3192 else
3193 sp = lookup_one(mp, name);
3194
3195 if (sp == NULL)
3196 return (0);
3197
3198 *size = (int)sp->st_size;
3199 return ((uintptr_t)sp->st_value);
3200 }
3201
3202 uintptr_t
kobj_lookup(struct module * mod,const char * name)3203 kobj_lookup(struct module *mod, const char *name)
3204 {
3205 Sym *sp;
3206
3207 sp = lookup_one(mod, name);
3208
3209 if (sp == NULL)
3210 return (0);
3211
3212 return ((uintptr_t)sp->st_value);
3213 }
3214
3215 char *
kobj_searchsym(struct module * mp,uintptr_t value,ulong_t * offset)3216 kobj_searchsym(struct module *mp, uintptr_t value, ulong_t *offset)
3217 {
3218 Sym *symtabptr;
3219 char *strtabptr;
3220 int symnum;
3221 Sym *sym;
3222 Sym *cursym;
3223 uintptr_t curval;
3224
3225 *offset = (ulong_t)-1l; /* assume not found */
3226 cursym = NULL;
3227
3228 if (kobj_addrcheck(mp, (void *)value) != 0)
3229 return (NULL); /* not in this module */
3230
3231 strtabptr = mp->strings;
3232 symtabptr = (Sym *)mp->symtbl;
3233
3234 /*
3235 * Scan the module's symbol table for a symbol <= value
3236 */
3237 for (symnum = 1, sym = symtabptr + 1;
3238 symnum < mp->nsyms; symnum++, sym = (Sym *)
3239 ((uintptr_t)sym + mp->symhdr->sh_entsize)) {
3240 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL) {
3241 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL)
3242 continue;
3243 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT &&
3244 ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3245 continue;
3246 }
3247
3248 curval = (uintptr_t)sym->st_value;
3249
3250 if (curval > value)
3251 continue;
3252
3253 /*
3254 * If one or both are functions...
3255 */
3256 if (ELF_ST_TYPE(sym->st_info) == STT_FUNC || (cursym != NULL &&
3257 ELF_ST_TYPE(cursym->st_info) == STT_FUNC)) {
3258 /* Ignore if the address is out of the bounds */
3259 if (value - sym->st_value >= sym->st_size)
3260 continue;
3261
3262 if (cursym != NULL &&
3263 ELF_ST_TYPE(cursym->st_info) == STT_FUNC) {
3264 /* Prefer the function to the non-function */
3265 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
3266 continue;
3267
3268 /* Prefer the larger of the two functions */
3269 if (sym->st_size <= cursym->st_size)
3270 continue;
3271 }
3272 } else if (value - curval >= *offset) {
3273 continue;
3274 }
3275
3276 *offset = (ulong_t)(value - curval);
3277 cursym = sym;
3278 }
3279 if (cursym == NULL)
3280 return (NULL);
3281
3282 return (strtabptr + cursym->st_name);
3283 }
3284
3285 Sym *
kobj_lookup_all(struct module * mp,char * name,int include_self)3286 kobj_lookup_all(struct module *mp, char *name, int include_self)
3287 {
3288 Sym *sp;
3289 struct module_list *mlp;
3290 struct modctl_list *clp;
3291 struct module *mmp;
3292
3293 if (include_self && (sp = lookup_one(mp, name)) != NULL)
3294 return (sp);
3295
3296 for (mlp = mp->head; mlp; mlp = mlp->next) {
3297 if ((sp = lookup_one(mlp->mp, name)) != NULL &&
3298 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3299 return (sp);
3300 }
3301
3302 /*
3303 * Loop through the primary kernel modules.
3304 */
3305 for (clp = kobj_lm_lookup(KOBJ_LM_PRIMARY); clp; clp = clp->modl_next) {
3306 mmp = mod(clp);
3307
3308 if (mmp == NULL || mp == mmp)
3309 continue;
3310
3311 if ((sp = lookup_one(mmp, name)) != NULL &&
3312 ELF_ST_BIND(sp->st_info) != STB_LOCAL)
3313 return (sp);
3314 }
3315 return (NULL);
3316 }
3317
3318 Sym *
kobj_lookup_kernel(const char * name)3319 kobj_lookup_kernel(const char *name)
3320 {
3321 struct modctl_list *lp;
3322 struct module *mp;
3323 Sym *sp;
3324
3325 /*
3326 * Loop through the primary kernel modules.
3327 */
3328 for (lp = kobj_lm_lookup(KOBJ_LM_PRIMARY); lp; lp = lp->modl_next) {
3329 mp = mod(lp);
3330
3331 if (mp == NULL)
3332 continue;
3333
3334 if ((sp = lookup_one(mp, name)) != NULL)
3335 return (sp);
3336 }
3337 return (NULL);
3338 }
3339
3340 static Sym *
lookup_one(struct module * mp,const char * name)3341 lookup_one(struct module *mp, const char *name)
3342 {
3343 symid_t *ip;
3344 char *name1;
3345 Sym *sp;
3346
3347 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3348 ip = &mp->chains[*ip]) {
3349 sp = (Sym *)(mp->symtbl +
3350 mp->symhdr->sh_entsize * *ip);
3351 name1 = mp->strings + sp->st_name;
3352 if (strcmp(name, name1) == 0 &&
3353 ELF_ST_TYPE(sp->st_info) != STT_FILE &&
3354 sp->st_shndx != SHN_UNDEF &&
3355 sp->st_shndx != SHN_COMMON)
3356 return (sp);
3357 }
3358 return (NULL);
3359 }
3360
3361 /*
3362 * Lookup a given symbol pointer in the module's symbol hash. If the symbol
3363 * is hashed, return the symbol pointer; otherwise return NULL.
3364 */
3365 static Sym *
sym_lookup(struct module * mp,Sym * ksp)3366 sym_lookup(struct module *mp, Sym *ksp)
3367 {
3368 char *name = mp->strings + ksp->st_name;
3369 symid_t *ip;
3370 Sym *sp;
3371
3372 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3373 ip = &mp->chains[*ip]) {
3374 sp = (Sym *)(mp->symtbl + mp->symhdr->sh_entsize * *ip);
3375 if (sp == ksp)
3376 return (ksp);
3377 }
3378 return (NULL);
3379 }
3380
3381 static void
sym_insert(struct module * mp,char * name,symid_t index)3382 sym_insert(struct module *mp, char *name, symid_t index)
3383 {
3384 symid_t *ip;
3385
3386 #ifdef KOBJ_DEBUG
3387 if (kobj_debug & D_SYMBOLS) {
3388 static struct module *lastmp = NULL;
3389 Sym *sp;
3390 if (lastmp != mp) {
3391 _kobj_printf(ops,
3392 "krtld: symbol entry: file=%s\n",
3393 mp->filename);
3394 _kobj_printf(ops,
3395 "krtld:\tsymndx\tvalue\t\t"
3396 "symbol name\n");
3397 lastmp = mp;
3398 }
3399 sp = (Sym *)(mp->symtbl +
3400 index * mp->symhdr->sh_entsize);
3401 _kobj_printf(ops, "krtld:\t[%3d]", index);
3402 _kobj_printf(ops, "\t0x%lx", sp->st_value);
3403 _kobj_printf(ops, "\t%s\n", name);
3404 }
3405 #endif
3406
3407 for (ip = &mp->buckets[kobj_hash_name(name) % mp->hashsize]; *ip;
3408 ip = &mp->chains[*ip]) {
3409 ;
3410 }
3411 *ip = index;
3412 }
3413
3414 struct modctl *
kobj_boot_mod_lookup(const char * modname)3415 kobj_boot_mod_lookup(const char *modname)
3416 {
3417 struct modctl *mctl = kobj_modules;
3418
3419 do {
3420 if (strcmp(modname, mctl->mod_modname) == 0)
3421 return (mctl);
3422 } while ((mctl = mctl->mod_next) != kobj_modules);
3423
3424 return (NULL);
3425 }
3426
3427 /*
3428 * Determine if the module exists.
3429 */
3430 int
kobj_path_exists(char * name,int use_path)3431 kobj_path_exists(char *name, int use_path)
3432 {
3433 struct _buf *file;
3434
3435 file = kobj_open_path(name, use_path, 1);
3436 #ifdef MODDIR_SUFFIX
3437 if (file == (struct _buf *)-1)
3438 file = kobj_open_path(name, use_path, 0);
3439 #endif /* MODDIR_SUFFIX */
3440 if (file == (struct _buf *)-1)
3441 return (0);
3442 kobj_close_file(file);
3443 return (1);
3444 }
3445
3446 /*
3447 * fullname is dynamically allocated to be able to hold the
3448 * maximum size string that can be constructed from name.
3449 * path is exactly like the shell PATH variable.
3450 */
3451 struct _buf *
kobj_open_path(char * name,int use_path,int use_moddir_suffix)3452 kobj_open_path(char *name, int use_path, int use_moddir_suffix)
3453 {
3454 char *p, *q;
3455 char *pathp;
3456 char *pathpsave;
3457 char *fullname;
3458 int maxpathlen;
3459 struct _buf *file;
3460
3461 #if !defined(MODDIR_SUFFIX)
3462 use_moddir_suffix = B_FALSE;
3463 #endif
3464
3465 if (!use_path)
3466 pathp = ""; /* use name as specified */
3467 else
3468 pathp = kobj_module_path;
3469 /* use configured default path */
3470
3471 pathpsave = pathp; /* keep this for error reporting */
3472
3473 /*
3474 * Allocate enough space for the largest possible fullname.
3475 * since path is of the form <directory> : <directory> : ...
3476 * we're potentially allocating a little more than we need to
3477 * but we'll allocate the exact amount when we find the right directory.
3478 * (The + 3 below is one for NULL terminator and one for the '/'
3479 * we might have to add at the beginning of path and one for
3480 * the '/' between path and name.)
3481 */
3482 maxpathlen = strlen(pathp) + strlen(name) + 3;
3483 /* sizeof includes null */
3484 maxpathlen += sizeof (slash_moddir_suffix_slash) - 1;
3485 fullname = kobj_zalloc(maxpathlen, KM_WAIT);
3486
3487 for (;;) {
3488 p = fullname;
3489 if (*pathp != '\0' && *pathp != '/')
3490 *p++ = '/'; /* path must start with '/' */
3491 while (*pathp && *pathp != ':' && *pathp != ' ')
3492 *p++ = *pathp++;
3493 if (p != fullname && p[-1] != '/')
3494 *p++ = '/';
3495 if (use_moddir_suffix) {
3496 char *b = basename(name);
3497 char *s;
3498
3499 /* copy everything up to the base name */
3500 q = name;
3501 while (q != b && *q)
3502 *p++ = *q++;
3503 s = slash_moddir_suffix_slash;
3504 while (*s)
3505 *p++ = *s++;
3506 /* copy the rest */
3507 while (*b)
3508 *p++ = *b++;
3509 } else {
3510 q = name;
3511 while (*q)
3512 *p++ = *q++;
3513 }
3514 *p = 0;
3515 if ((file = kobj_open_file(fullname)) != (struct _buf *)-1) {
3516 kobj_free(fullname, maxpathlen);
3517 return (file);
3518 }
3519 while (*pathp == ' ' || *pathp == ':')
3520 pathp++;
3521 if (*pathp == 0)
3522 break;
3523
3524 }
3525 kobj_free(fullname, maxpathlen);
3526 if (_moddebug & MODDEBUG_ERRMSG) {
3527 _kobj_printf(ops, "can't open %s,", name);
3528 _kobj_printf(ops, " path is %s\n", pathpsave);
3529 }
3530 return ((struct _buf *)-1);
3531 }
3532
3533 intptr_t
kobj_open(char * filename)3534 kobj_open(char *filename)
3535 {
3536 struct vnode *vp;
3537 int fd;
3538
3539 if (_modrootloaded) {
3540 struct kobjopen_tctl *ltp = kobjopen_alloc(filename);
3541 int Errno;
3542
3543 /*
3544 * Hand off the open to a thread who has a
3545 * stack size capable handling the request.
3546 */
3547 if (curthread != &t0) {
3548 (void) thread_create(NULL, DEFAULTSTKSZ * 2,
3549 kobjopen_thread, ltp, 0, &p0, TS_RUN, maxclsyspri);
3550 sema_p(<p->sema);
3551 Errno = ltp->Errno;
3552 vp = ltp->vp;
3553 } else {
3554 /*
3555 * 1098067: module creds should not be those of the
3556 * caller
3557 */
3558 cred_t *saved_cred = curthread->t_cred;
3559 curthread->t_cred = kcred;
3560 Errno = vn_openat(filename, UIO_SYSSPACE, FREAD, 0, &vp,
3561 0, 0, rootdir, -1);
3562 curthread->t_cred = saved_cred;
3563 }
3564 kobjopen_free(ltp);
3565
3566 if (Errno) {
3567 if (_moddebug & MODDEBUG_ERRMSG) {
3568 _kobj_printf(ops,
3569 "kobj_open: vn_open of %s fails, ",
3570 filename);
3571 _kobj_printf(ops, "Errno = %d\n", Errno);
3572 }
3573 return (-1);
3574 } else {
3575 if (_moddebug & MODDEBUG_ERRMSG) {
3576 _kobj_printf(ops, "kobj_open: '%s'", filename);
3577 _kobj_printf(ops, " vp = %p\n", vp);
3578 }
3579 return ((intptr_t)vp);
3580 }
3581 } else {
3582 fd = kobj_boot_open(filename, 0);
3583
3584 if (_moddebug & MODDEBUG_ERRMSG) {
3585 if (fd < 0)
3586 _kobj_printf(ops,
3587 "kobj_open: can't open %s\n", filename);
3588 else {
3589 _kobj_printf(ops, "kobj_open: '%s'", filename);
3590 _kobj_printf(ops, " descr = 0x%x\n", fd);
3591 }
3592 }
3593 return ((intptr_t)fd);
3594 }
3595 }
3596
3597 /*
3598 * Calls to kobj_open() are handled off to this routine as a separate thread.
3599 */
3600 static void
kobjopen_thread(struct kobjopen_tctl * ltp)3601 kobjopen_thread(struct kobjopen_tctl *ltp)
3602 {
3603 kmutex_t cpr_lk;
3604 callb_cpr_t cpr_i;
3605
3606 mutex_init(&cpr_lk, NULL, MUTEX_DEFAULT, NULL);
3607 CALLB_CPR_INIT(&cpr_i, &cpr_lk, callb_generic_cpr, "kobjopen");
3608 ltp->Errno = vn_open(ltp->name, UIO_SYSSPACE, FREAD, 0, &(ltp->vp),
3609 0, 0);
3610 sema_v(<p->sema);
3611 mutex_enter(&cpr_lk);
3612 CALLB_CPR_EXIT(&cpr_i);
3613 mutex_destroy(&cpr_lk);
3614 thread_exit();
3615 }
3616
3617 /*
3618 * allocate and initialize a kobjopen thread structure
3619 */
3620 static struct kobjopen_tctl *
kobjopen_alloc(char * filename)3621 kobjopen_alloc(char *filename)
3622 {
3623 struct kobjopen_tctl *ltp = kmem_zalloc(sizeof (*ltp), KM_SLEEP);
3624
3625 ASSERT(filename != NULL);
3626
3627 ltp->name = kmem_alloc(strlen(filename) + 1, KM_SLEEP);
3628 bcopy(filename, ltp->name, strlen(filename) + 1);
3629 sema_init(<p->sema, 0, NULL, SEMA_DEFAULT, NULL);
3630 return (ltp);
3631 }
3632
3633 /*
3634 * free a kobjopen thread control structure
3635 */
3636 static void
kobjopen_free(struct kobjopen_tctl * ltp)3637 kobjopen_free(struct kobjopen_tctl *ltp)
3638 {
3639 sema_destroy(<p->sema);
3640 kmem_free(ltp->name, strlen(ltp->name) + 1);
3641 kmem_free(ltp, sizeof (*ltp));
3642 }
3643
3644 int
kobj_read(intptr_t descr,char * buf,uint_t size,uint_t offset)3645 kobj_read(intptr_t descr, char *buf, uint_t size, uint_t offset)
3646 {
3647 int stat;
3648 ssize_t resid;
3649
3650 if (_modrootloaded) {
3651 if ((stat = vn_rdwr(UIO_READ, (struct vnode *)descr, buf, size,
3652 (offset_t)offset, UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3653 &resid)) != 0) {
3654 _kobj_printf(ops,
3655 "vn_rdwr failed with error 0x%x\n", stat);
3656 return (-1);
3657 }
3658 return (size - resid);
3659 } else {
3660 int count = 0;
3661
3662 if (kobj_boot_seek((int)descr, (off_t)0, offset) != 0) {
3663 _kobj_printf(ops,
3664 "kobj_read: seek 0x%x failed\n", offset);
3665 return (-1);
3666 }
3667
3668 count = kobj_boot_read((int)descr, buf, size);
3669 if (count < size) {
3670 if (_moddebug & MODDEBUG_ERRMSG) {
3671 _kobj_printf(ops,
3672 "kobj_read: req %d bytes, ", size);
3673 _kobj_printf(ops, "got %d\n", count);
3674 }
3675 }
3676 return (count);
3677 }
3678 }
3679
3680 void
kobj_close(intptr_t descr)3681 kobj_close(intptr_t descr)
3682 {
3683 if (_moddebug & MODDEBUG_ERRMSG)
3684 _kobj_printf(ops, "kobj_close: 0x%lx\n", descr);
3685
3686 if (_modrootloaded) {
3687 struct vnode *vp = (struct vnode *)descr;
3688 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
3689 VN_RELE(vp);
3690 } else
3691 (void) kobj_boot_close((int)descr);
3692 }
3693
3694 int
kobj_fstat(intptr_t descr,struct bootstat * buf)3695 kobj_fstat(intptr_t descr, struct bootstat *buf)
3696 {
3697 if (buf == NULL)
3698 return (-1);
3699
3700 if (_modrootloaded) {
3701 vattr_t vattr;
3702 struct vnode *vp = (struct vnode *)descr;
3703 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3704 return (-1);
3705
3706 /*
3707 * The vattr and bootstat structures are similar, but not
3708 * identical. We do our best to fill in the bootstat structure
3709 * from the contents of vattr (transfering only the ones that
3710 * are obvious.
3711 */
3712
3713 buf->st_mode = (uint32_t)vattr.va_mode;
3714 buf->st_nlink = (uint32_t)vattr.va_nlink;
3715 buf->st_uid = (int32_t)vattr.va_uid;
3716 buf->st_gid = (int32_t)vattr.va_gid;
3717 buf->st_rdev = (uint64_t)vattr.va_rdev;
3718 buf->st_size = (uint64_t)vattr.va_size;
3719 buf->st_atim.tv_sec = (int64_t)vattr.va_atime.tv_sec;
3720 buf->st_atim.tv_nsec = (int64_t)vattr.va_atime.tv_nsec;
3721 buf->st_mtim.tv_sec = (int64_t)vattr.va_mtime.tv_sec;
3722 buf->st_mtim.tv_nsec = (int64_t)vattr.va_mtime.tv_nsec;
3723 buf->st_ctim.tv_sec = (int64_t)vattr.va_ctime.tv_sec;
3724 buf->st_ctim.tv_nsec = (int64_t)vattr.va_ctime.tv_nsec;
3725 buf->st_blksize = (int32_t)vattr.va_blksize;
3726 buf->st_blocks = (int64_t)vattr.va_nblocks;
3727
3728 return (0);
3729 }
3730
3731 return (kobj_boot_fstat((int)descr, buf));
3732 }
3733
3734
3735 struct _buf *
kobj_open_file(char * name)3736 kobj_open_file(char *name)
3737 {
3738 struct _buf *file;
3739 struct compinfo cbuf;
3740 intptr_t fd;
3741
3742 if ((fd = kobj_open(name)) == -1) {
3743 return ((struct _buf *)-1);
3744 }
3745
3746 file = kobj_zalloc(sizeof (struct _buf), KM_WAIT|KM_TMP);
3747 file->_fd = fd;
3748 file->_name = kobj_alloc(strlen(name)+1, KM_WAIT|KM_TMP);
3749 file->_cnt = file->_size = file->_off = 0;
3750 file->_ln = 1;
3751 file->_ptr = file->_base;
3752 (void) strcpy(file->_name, name);
3753
3754 /*
3755 * Before root is mounted, we must check
3756 * for a compressed file and do our own
3757 * buffering.
3758 */
3759 if (_modrootloaded) {
3760 file->_base = kobj_zalloc(MAXBSIZE, KM_WAIT);
3761 file->_bsize = MAXBSIZE;
3762
3763 /* Check if the file is compressed */
3764 file->_iscmp = kobj_is_compressed(fd);
3765 } else {
3766 if (kobj_boot_compinfo(fd, &cbuf) != 0) {
3767 kobj_close_file(file);
3768 return ((struct _buf *)-1);
3769 }
3770 file->_iscmp = cbuf.iscmp;
3771 if (file->_iscmp) {
3772 if (kobj_comp_setup(file, &cbuf) != 0) {
3773 kobj_close_file(file);
3774 return ((struct _buf *)-1);
3775 }
3776 } else {
3777 file->_base = kobj_zalloc(cbuf.blksize, KM_WAIT|KM_TMP);
3778 file->_bsize = cbuf.blksize;
3779 }
3780 }
3781 return (file);
3782 }
3783
3784 static int
kobj_comp_setup(struct _buf * file,struct compinfo * cip)3785 kobj_comp_setup(struct _buf *file, struct compinfo *cip)
3786 {
3787 struct comphdr *hdr;
3788
3789 /*
3790 * read the compressed image into memory,
3791 * so we can deompress from there
3792 */
3793 file->_dsize = cip->fsize;
3794 file->_dbuf = kobj_alloc(cip->fsize, KM_WAIT|KM_TMP);
3795 if (kobj_read(file->_fd, file->_dbuf, cip->fsize, 0) != cip->fsize) {
3796 kobj_free(file->_dbuf, cip->fsize);
3797 return (-1);
3798 }
3799
3800 hdr = kobj_comphdr(file);
3801 if (hdr->ch_magic != CH_MAGIC_ZLIB || hdr->ch_version != CH_VERSION ||
3802 hdr->ch_algorithm != CH_ALG_ZLIB || hdr->ch_fsize == 0 ||
3803 !ISP2(hdr->ch_blksize)) {
3804 kobj_free(file->_dbuf, cip->fsize);
3805 return (-1);
3806 }
3807 file->_base = kobj_alloc(hdr->ch_blksize, KM_WAIT|KM_TMP);
3808 file->_bsize = hdr->ch_blksize;
3809 return (0);
3810 }
3811
3812 void
kobj_close_file(struct _buf * file)3813 kobj_close_file(struct _buf *file)
3814 {
3815 kobj_close(file->_fd);
3816 if (file->_base != NULL)
3817 kobj_free(file->_base, file->_bsize);
3818 if (file->_dbuf != NULL)
3819 kobj_free(file->_dbuf, file->_dsize);
3820 kobj_free(file->_name, strlen(file->_name)+1);
3821 kobj_free(file, sizeof (struct _buf));
3822 }
3823
3824 int
kobj_read_file(struct _buf * file,char * buf,uint_t size,uint_t off)3825 kobj_read_file(struct _buf *file, char *buf, uint_t size, uint_t off)
3826 {
3827 int b_size, c_size;
3828 int b_off; /* Offset into buffer for start of bcopy */
3829 int count = 0;
3830 int page_addr;
3831
3832 if (_moddebug & MODDEBUG_ERRMSG) {
3833 _kobj_printf(ops, "kobj_read_file: size=%x,", size);
3834 _kobj_printf(ops, " offset=%x at", off);
3835 _kobj_printf(ops, " buf=%lx\n", (uintptr_t)buf);
3836 }
3837
3838 /*
3839 * Handle compressed (gzip for now) file here. First get the
3840 * compressed size, then read the image into memory and finally
3841 * call zlib to decompress the image at the supplied memory buffer.
3842 */
3843 if (file->_iscmp == CH_MAGIC_GZIP) {
3844 ulong_t dlen;
3845 vattr_t vattr;
3846 struct vnode *vp = (struct vnode *)file->_fd;
3847 ssize_t resid;
3848 int err = 0;
3849
3850 if (VOP_GETATTR(vp, &vattr, 0, kcred, NULL) != 0)
3851 return (-1);
3852
3853 file->_dbuf = kobj_alloc(vattr.va_size, KM_WAIT|KM_TMP);
3854 file->_dsize = vattr.va_size;
3855
3856 /* Read the compressed file into memory */
3857 if ((err = vn_rdwr(UIO_READ, vp, file->_dbuf, vattr.va_size,
3858 (offset_t)(0), UIO_SYSSPACE, 0, (rlim64_t)0, CRED(),
3859 &resid)) != 0) {
3860
3861 _kobj_printf(ops, "kobj_read_file :vn_rdwr() failed, "
3862 "error code 0x%x\n", err);
3863 return (-1);
3864 }
3865
3866 dlen = size;
3867
3868 /* Decompress the image at the supplied memory buffer */
3869 if ((err = z_uncompress(buf, &dlen, file->_dbuf,
3870 vattr.va_size)) != Z_OK) {
3871 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3872 "failed, error code : 0x%x\n", err);
3873 return (-1);
3874 }
3875
3876 if (dlen != size) {
3877 _kobj_printf(ops, "kobj_read_file: z_uncompress "
3878 "failed to uncompress (size returned 0x%lx , "
3879 "expected size: 0x%x)\n", dlen, size);
3880 return (-1);
3881 }
3882
3883 return (0);
3884 }
3885
3886 while (size) {
3887 page_addr = F_PAGE(file, off);
3888 b_size = file->_size;
3889 /*
3890 * If we have the filesystem page the caller's referring to
3891 * and we have something in the buffer,
3892 * satisfy as much of the request from the buffer as we can.
3893 */
3894 if (page_addr == file->_off && b_size > 0) {
3895 b_off = B_OFFSET(file, off);
3896 c_size = b_size - b_off;
3897 /*
3898 * If there's nothing to copy, we're at EOF.
3899 */
3900 if (c_size <= 0)
3901 break;
3902 if (c_size > size)
3903 c_size = size;
3904 if (buf) {
3905 if (_moddebug & MODDEBUG_ERRMSG)
3906 _kobj_printf(ops, "copying %x bytes\n",
3907 c_size);
3908 bcopy(file->_base+b_off, buf, c_size);
3909 size -= c_size;
3910 off += c_size;
3911 buf += c_size;
3912 count += c_size;
3913 } else {
3914 _kobj_printf(ops, "kobj_read: system error");
3915 count = -1;
3916 break;
3917 }
3918 } else {
3919 /*
3920 * If the caller's offset is page aligned and
3921 * the caller want's at least a filesystem page and
3922 * the caller provided a buffer,
3923 * read directly into the caller's buffer.
3924 */
3925 if (page_addr == off &&
3926 (c_size = F_BLKS(file, size)) && buf) {
3927 c_size = kobj_read_blks(file, buf, c_size,
3928 page_addr);
3929 if (c_size < 0) {
3930 count = -1;
3931 break;
3932 }
3933 count += c_size;
3934 if (c_size != F_BLKS(file, size))
3935 break;
3936 size -= c_size;
3937 off += c_size;
3938 buf += c_size;
3939 /*
3940 * Otherwise, read into our buffer and copy next time
3941 * around the loop.
3942 */
3943 } else {
3944 file->_off = page_addr;
3945 c_size = kobj_read_blks(file, file->_base,
3946 file->_bsize, page_addr);
3947 file->_ptr = file->_base;
3948 file->_cnt = c_size;
3949 file->_size = c_size;
3950 /*
3951 * If a _filbuf call or nothing read, break.
3952 */
3953 if (buf == NULL || c_size <= 0) {
3954 count = c_size;
3955 break;
3956 }
3957 }
3958 if (_moddebug & MODDEBUG_ERRMSG)
3959 _kobj_printf(ops, "read %x bytes\n", c_size);
3960 }
3961 }
3962 if (_moddebug & MODDEBUG_ERRMSG)
3963 _kobj_printf(ops, "count = %x\n", count);
3964
3965 return (count);
3966 }
3967
3968 static int
kobj_read_blks(struct _buf * file,char * buf,uint_t size,uint_t off)3969 kobj_read_blks(struct _buf *file, char *buf, uint_t size, uint_t off)
3970 {
3971 int ret;
3972
3973 ASSERT(B_OFFSET(file, size) == 0 && B_OFFSET(file, off) == 0);
3974 if (file->_iscmp) {
3975 uint_t blks;
3976 int nret;
3977
3978 ret = 0;
3979 for (blks = size / file->_bsize; blks != 0; blks--) {
3980 nret = kobj_uncomp_blk(file, buf, off);
3981 if (nret == -1)
3982 return (-1);
3983 buf += nret;
3984 off += nret;
3985 ret += nret;
3986 if (nret < file->_bsize)
3987 break;
3988 }
3989 } else
3990 ret = kobj_read(file->_fd, buf, size, off);
3991 return (ret);
3992 }
3993
3994 static int
kobj_uncomp_blk(struct _buf * file,char * buf,uint_t off)3995 kobj_uncomp_blk(struct _buf *file, char *buf, uint_t off)
3996 {
3997 struct comphdr *hdr = kobj_comphdr(file);
3998 ulong_t dlen, slen;
3999 caddr_t src;
4000 int i;
4001
4002 dlen = file->_bsize;
4003 i = off / file->_bsize;
4004 src = file->_dbuf + hdr->ch_blkmap[i];
4005 if (i == hdr->ch_fsize / file->_bsize)
4006 slen = file->_dsize - hdr->ch_blkmap[i];
4007 else
4008 slen = hdr->ch_blkmap[i + 1] - hdr->ch_blkmap[i];
4009 if (z_uncompress(buf, &dlen, src, slen) != Z_OK)
4010 return (-1);
4011 return (dlen);
4012 }
4013
4014 int
kobj_filbuf(struct _buf * f)4015 kobj_filbuf(struct _buf *f)
4016 {
4017 if (kobj_read_file(f, NULL, f->_bsize, f->_off + f->_size) > 0)
4018 return (kobj_getc(f));
4019 return (-1);
4020 }
4021
4022 void
kobj_free(void * address,size_t size)4023 kobj_free(void *address, size_t size)
4024 {
4025 if (standalone)
4026 return;
4027
4028 kmem_free(address, size);
4029 kobj_stat.nfree_calls++;
4030 kobj_stat.nfree += size;
4031 }
4032
4033 void *
kobj_zalloc(size_t size,int flag)4034 kobj_zalloc(size_t size, int flag)
4035 {
4036 void *v;
4037
4038 if ((v = kobj_alloc(size, flag)) != 0) {
4039 bzero(v, size);
4040 }
4041
4042 return (v);
4043 }
4044
4045 void *
kobj_alloc(size_t size,int flag)4046 kobj_alloc(size_t size, int flag)
4047 {
4048 /*
4049 * If we are running standalone in the
4050 * linker, we ask boot for memory.
4051 * Either it's temporary memory that we lose
4052 * once boot is mapped out or we allocate it
4053 * permanently using the dynamic data segment.
4054 */
4055 if (standalone) {
4056 #if defined(_OBP)
4057 if (flag & (KM_TMP | KM_SCRATCH))
4058 return (bop_temp_alloc(size, MINALIGN));
4059 #else
4060 if (flag & (KM_TMP | KM_SCRATCH))
4061 return (BOP_ALLOC(ops, 0, size, MINALIGN));
4062 #endif
4063 return (kobj_segbrk(&_edata, size, MINALIGN, 0));
4064 }
4065
4066 kobj_stat.nalloc_calls++;
4067 kobj_stat.nalloc += size;
4068
4069 return (kmem_alloc(size, (flag & KM_NOWAIT) ? KM_NOSLEEP : KM_SLEEP));
4070 }
4071
4072 /*
4073 * Allow the "mod" system to sync up with the work
4074 * already done by kobj during the initial loading
4075 * of the kernel. This also gives us a chance
4076 * to reallocate memory that belongs to boot.
4077 */
4078 void
kobj_sync(void)4079 kobj_sync(void)
4080 {
4081 struct modctl_list *lp, **lpp;
4082
4083 /*
4084 * The module path can be set in /etc/system via 'moddir' commands
4085 */
4086 if (default_path != NULL)
4087 kobj_module_path = default_path;
4088 else
4089 default_path = kobj_module_path;
4090
4091 ksyms_arena = vmem_create("ksyms", NULL, 0, sizeof (uint64_t),
4092 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4093
4094 ctf_arena = vmem_create("ctf", NULL, 0, sizeof (uint_t),
4095 segkmem_alloc, segkmem_free, heap_arena, 0, VM_SLEEP);
4096
4097 /*
4098 * Move symbol tables from boot memory to ksyms_arena.
4099 */
4100 for (lpp = kobj_linkmaps; *lpp != NULL; lpp++) {
4101 for (lp = *lpp; lp != NULL; lp = lp->modl_next)
4102 kobj_export_module(mod(lp));
4103 }
4104 }
4105
4106 caddr_t
kobj_segbrk(caddr_t * spp,size_t size,size_t align,caddr_t limit)4107 kobj_segbrk(caddr_t *spp, size_t size, size_t align, caddr_t limit)
4108 {
4109 uintptr_t va, pva;
4110 size_t alloc_pgsz = kobj_mmu_pagesize;
4111 size_t alloc_align = BO_NO_ALIGN;
4112 size_t alloc_size;
4113
4114 /*
4115 * If we are using "large" mappings for the kernel,
4116 * request aligned memory from boot using the
4117 * "large" pagesize.
4118 */
4119 if (lg_pagesize) {
4120 alloc_align = lg_pagesize;
4121 alloc_pgsz = lg_pagesize;
4122 }
4123
4124 #if defined(__sparc)
4125 /* account for redzone */
4126 if (limit)
4127 limit -= alloc_pgsz;
4128 #endif /* __sparc */
4129
4130 va = ALIGN((uintptr_t)*spp, align);
4131 pva = P2ROUNDUP((uintptr_t)*spp, alloc_pgsz);
4132 /*
4133 * Need more pages?
4134 */
4135 if (va + size > pva) {
4136 uintptr_t npva;
4137
4138 alloc_size = P2ROUNDUP(size - (pva - va), alloc_pgsz);
4139 /*
4140 * Check for overlapping segments.
4141 */
4142 if (limit && limit <= *spp + alloc_size) {
4143 return ((caddr_t)0);
4144 }
4145
4146 npva = (uintptr_t)BOP_ALLOC(ops, (caddr_t)pva,
4147 alloc_size, alloc_align);
4148
4149 if (npva == 0) {
4150 _kobj_printf(ops, "BOP_ALLOC failed, 0x%lx bytes",
4151 alloc_size);
4152 _kobj_printf(ops, " aligned %lx", alloc_align);
4153 _kobj_printf(ops, " at 0x%lx\n", pva);
4154 return (NULL);
4155 }
4156 }
4157 *spp = (caddr_t)(va + size);
4158
4159 return ((caddr_t)va);
4160 }
4161
4162 /*
4163 * Calculate the number of output hash buckets.
4164 * We use the next prime larger than n / 4,
4165 * so the average hash chain is about 4 entries.
4166 * More buckets would just be a waste of memory.
4167 */
4168 uint_t
kobj_gethashsize(uint_t n)4169 kobj_gethashsize(uint_t n)
4170 {
4171 int f;
4172 int hsize = MAX(n / 4, 2);
4173
4174 for (f = 2; f * f <= hsize; f++)
4175 if (hsize % f == 0)
4176 hsize += f = 1;
4177
4178 return (hsize);
4179 }
4180
4181 /*
4182 * Get the file size.
4183 *
4184 * Before root is mounted, files are compressed in the boot_archive ramdisk
4185 * (in the memory). kobj_fstat would return the compressed file size.
4186 * In order to get the uncompressed file size, read the file to the end and
4187 * count its size.
4188 */
4189 int
kobj_get_filesize(struct _buf * file,uint64_t * size)4190 kobj_get_filesize(struct _buf *file, uint64_t *size)
4191 {
4192 int err = 0;
4193 ssize_t resid;
4194 uint32_t buf;
4195
4196 if (_modrootloaded) {
4197 struct bootstat bst;
4198
4199 if (kobj_fstat(file->_fd, &bst) != 0)
4200 return (EIO);
4201 *size = bst.st_size;
4202
4203 if (file->_iscmp == CH_MAGIC_GZIP) {
4204 /*
4205 * Read the last 4 bytes of the compressed (gzip)
4206 * image to get the size of its uncompressed
4207 * version.
4208 */
4209 if ((err = vn_rdwr(UIO_READ, (struct vnode *)file->_fd,
4210 (char *)(&buf), 4, (offset_t)(*size - 4),
4211 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid))
4212 != 0) {
4213 _kobj_printf(ops, "kobj_get_filesize: "
4214 "vn_rdwr() failed with error 0x%x\n", err);
4215 return (-1);
4216 }
4217
4218 *size = (uint64_t)buf;
4219 }
4220 } else {
4221
4222 #if defined(_OBP)
4223 struct bootstat bsb;
4224
4225 if (file->_iscmp) {
4226 struct comphdr *hdr = kobj_comphdr(file);
4227
4228 *size = hdr->ch_fsize;
4229 } else if (kobj_boot_fstat(file->_fd, &bsb) != 0)
4230 return (EIO);
4231 else
4232 *size = bsb.st_size;
4233 #else
4234 char *buf;
4235 int count;
4236 uint64_t offset = 0;
4237
4238 buf = kmem_alloc(MAXBSIZE, KM_SLEEP);
4239 do {
4240 count = kobj_read_file(file, buf, MAXBSIZE, offset);
4241 if (count < 0) {
4242 kmem_free(buf, MAXBSIZE);
4243 return (EIO);
4244 }
4245 offset += count;
4246 } while (count == MAXBSIZE);
4247 kmem_free(buf, MAXBSIZE);
4248
4249 *size = offset;
4250 #endif
4251 }
4252
4253 return (0);
4254 }
4255
4256 static char *
basename(char * s)4257 basename(char *s)
4258 {
4259 char *p, *q;
4260
4261 q = NULL;
4262 p = s;
4263 do {
4264 if (*p == '/')
4265 q = p;
4266 } while (*p++);
4267 return (q ? q + 1 : s);
4268 }
4269
4270 void
kobj_stat_get(kobj_stat_t * kp)4271 kobj_stat_get(kobj_stat_t *kp)
4272 {
4273 *kp = kobj_stat;
4274 }
4275
4276 int
kobj_getpagesize()4277 kobj_getpagesize()
4278 {
4279 return (lg_pagesize);
4280 }
4281
4282 void
kobj_textwin_alloc(struct module * mp)4283 kobj_textwin_alloc(struct module *mp)
4284 {
4285 ASSERT(MUTEX_HELD(&mod_lock));
4286
4287 if (mp->textwin != NULL)
4288 return;
4289
4290 /*
4291 * If the text is not contained in the heap, then it is not contained
4292 * by a writable mapping. (Specifically, it's on the nucleus page.)
4293 * We allocate a read/write mapping for this module's text to allow
4294 * the text to be patched without calling hot_patch_kernel_text()
4295 * (which is quite slow).
4296 */
4297 if (!vmem_contains(heaptext_arena, mp->text, mp->text_size)) {
4298 uintptr_t text = (uintptr_t)mp->text;
4299 uintptr_t size = (uintptr_t)mp->text_size;
4300 uintptr_t i;
4301 caddr_t va;
4302 size_t sz = ((text + size + PAGESIZE - 1) & PAGEMASK) -
4303 (text & PAGEMASK);
4304
4305 va = mp->textwin_base = vmem_alloc(heap_arena, sz, VM_SLEEP);
4306
4307 for (i = text & PAGEMASK; i < text + size; i += PAGESIZE) {
4308 hat_devload(kas.a_hat, va, PAGESIZE,
4309 hat_getpfnum(kas.a_hat, (caddr_t)i),
4310 PROT_READ | PROT_WRITE,
4311 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
4312 va += PAGESIZE;
4313 }
4314
4315 mp->textwin = mp->textwin_base + (text & PAGEOFFSET);
4316 } else {
4317 mp->textwin = mp->text;
4318 }
4319 }
4320
4321 void
kobj_textwin_free(struct module * mp)4322 kobj_textwin_free(struct module *mp)
4323 {
4324 uintptr_t text = (uintptr_t)mp->text;
4325 uintptr_t tsize = (uintptr_t)mp->text_size;
4326 size_t size = (((text + tsize + PAGESIZE - 1) & PAGEMASK) -
4327 (text & PAGEMASK));
4328
4329 mp->textwin = NULL;
4330
4331 if (mp->textwin_base == NULL)
4332 return;
4333
4334 hat_unload(kas.a_hat, mp->textwin_base, size, HAT_UNLOAD_UNLOCK);
4335 vmem_free(heap_arena, mp->textwin_base, size);
4336 mp->textwin_base = NULL;
4337 }
4338
4339 static char *
find_libmacro(char * name)4340 find_libmacro(char *name)
4341 {
4342 int lmi;
4343
4344 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4345 if (strcmp(name, libmacros[lmi].lmi_macroname) == 0)
4346 return (libmacros[lmi].lmi_list);
4347 }
4348 return (NULL);
4349 }
4350
4351 /*
4352 * Check for $MACRO in tail (string to expand) and expand it in path at pathend
4353 * returns path if successful, else NULL
4354 * Support multiple $MACROs expansion and the first valid path will be returned
4355 * Caller's responsibility to provide enough space in path to expand
4356 */
4357 char *
expand_libmacro(char * tail,char * path,char * pathend)4358 expand_libmacro(char *tail, char *path, char *pathend)
4359 {
4360 char c, *p, *p1, *p2, *path2, *endp;
4361 int diff, lmi, macrolen, valid_macro, more_macro;
4362 struct _buf *file;
4363
4364 /*
4365 * check for $MACROS between nulls or slashes
4366 */
4367 p = strchr(tail, '$');
4368 if (p == NULL)
4369 return (NULL);
4370 for (lmi = 0; lmi < NLIBMACROS; lmi++) {
4371 macrolen = libmacros[lmi].lmi_macrolen;
4372 if (strncmp(p + 1, libmacros[lmi].lmi_macroname, macrolen) == 0)
4373 break;
4374 }
4375
4376 valid_macro = 0;
4377 if (lmi < NLIBMACROS) {
4378 /*
4379 * The following checks are used to restrict expansion of
4380 * macros to those that form a full directory/file name
4381 * and to keep the behavior same as before. If this
4382 * restriction is removed or no longer valid in the future,
4383 * the checks below can be deleted.
4384 */
4385 if ((p == tail) || (*(p - 1) == '/')) {
4386 c = *(p + macrolen + 1);
4387 if (c == '/' || c == '\0')
4388 valid_macro = 1;
4389 }
4390 }
4391
4392 if (!valid_macro) {
4393 p2 = strchr(p, '/');
4394 /*
4395 * if no more macro to expand, then just copy whatever left
4396 * and check whether it exists
4397 */
4398 if (p2 == NULL || strchr(p2, '$') == NULL) {
4399 (void) strcpy(pathend, tail);
4400 if ((file = kobj_open_path(path, 1, 1)) !=
4401 (struct _buf *)-1) {
4402 kobj_close_file(file);
4403 return (path);
4404 } else
4405 return (NULL);
4406 } else {
4407 /*
4408 * copy all chars before '/' and call expand_libmacro()
4409 * again
4410 */
4411 diff = p2 - tail;
4412 bcopy(tail, pathend, diff);
4413 pathend += diff;
4414 *(pathend) = '\0';
4415 return (expand_libmacro(p2, path, pathend));
4416 }
4417 }
4418
4419 more_macro = 0;
4420 if (c != '\0') {
4421 endp = p + macrolen + 1;
4422 if (strchr(endp, '$') != NULL)
4423 more_macro = 1;
4424 } else
4425 endp = NULL;
4426
4427 /*
4428 * copy lmi_list and split it into components.
4429 * then put the part of tail before $MACRO into path
4430 * at pathend
4431 */
4432 diff = p - tail;
4433 if (diff > 0)
4434 bcopy(tail, pathend, diff);
4435 path2 = pathend + diff;
4436 p1 = libmacros[lmi].lmi_list;
4437 while (p1 && (*p1 != '\0')) {
4438 p2 = strchr(p1, ':');
4439 if (p2) {
4440 diff = p2 - p1;
4441 bcopy(p1, path2, diff);
4442 *(path2 + diff) = '\0';
4443 } else {
4444 diff = strlen(p1);
4445 bcopy(p1, path2, diff + 1);
4446 }
4447 /* copy endp only if there isn't any more macro to expand */
4448 if (!more_macro && (endp != NULL))
4449 (void) strcat(path2, endp);
4450 file = kobj_open_path(path, 1, 1);
4451 if (file != (struct _buf *)-1) {
4452 kobj_close_file(file);
4453 /*
4454 * if more macros to expand then call expand_libmacro(),
4455 * else return path which has the whole path
4456 */
4457 if (!more_macro || (expand_libmacro(endp, path,
4458 path2 + diff) != NULL)) {
4459 return (path);
4460 }
4461 }
4462 if (p2)
4463 p1 = ++p2;
4464 else
4465 return (NULL);
4466 }
4467 return (NULL);
4468 }
4469
4470 char *kobj_file_buf;
4471 int kobj_file_bufsize;
4472
4473 /*
4474 * This code is for the purpose of manually recording which files
4475 * needs to go into the boot archive on any given system.
4476 *
4477 * To enable the code, set kobj_file_bufsize in /etc/system
4478 * and reboot the system, then use mdb to look at kobj_file_buf.
4479 */
4480 static void
kobj_record_file(char * filename)4481 kobj_record_file(char *filename)
4482 {
4483 static char *buf;
4484 static int size = 0;
4485 int n;
4486
4487 if (kobj_file_bufsize == 0) /* don't bother */
4488 return;
4489
4490 if (kobj_file_buf == NULL) { /* allocate buffer */
4491 size = kobj_file_bufsize;
4492 buf = kobj_file_buf = kobj_alloc(size, KM_WAIT|KM_TMP);
4493 }
4494
4495 n = snprintf(buf, size, "%s\n", filename);
4496 if (n > size)
4497 n = size;
4498 size -= n;
4499 buf += n;
4500 }
4501
4502 static int
kobj_boot_fstat(int fd,struct bootstat * stp)4503 kobj_boot_fstat(int fd, struct bootstat *stp)
4504 {
4505 #if defined(_OBP)
4506 if (!standalone && _ioquiesced)
4507 return (-1);
4508 return (BOP_FSTAT(ops, fd, stp));
4509 #else
4510 return (BRD_FSTAT(bfs_ops, fd, stp));
4511 #endif
4512 }
4513
4514 static int
kobj_boot_open(char * filename,int flags)4515 kobj_boot_open(char *filename, int flags)
4516 {
4517 #if defined(_OBP)
4518
4519 /*
4520 * If io via bootops is quiesced, it means boot is no longer
4521 * available to us. We make it look as if we can't open the
4522 * named file - which is reasonably accurate.
4523 */
4524 if (!standalone && _ioquiesced)
4525 return (-1);
4526
4527 kobj_record_file(filename);
4528 return (BOP_OPEN(filename, flags));
4529 #else /* x86 */
4530 kobj_record_file(filename);
4531 return (BRD_OPEN(bfs_ops, filename, flags));
4532 #endif
4533 }
4534
4535 static int
kobj_boot_close(int fd)4536 kobj_boot_close(int fd)
4537 {
4538 #if defined(_OBP)
4539 if (!standalone && _ioquiesced)
4540 return (-1);
4541
4542 return (BOP_CLOSE(fd));
4543 #else /* x86 */
4544 return (BRD_CLOSE(bfs_ops, fd));
4545 #endif
4546 }
4547
4548 /*ARGSUSED*/
4549 static int
kobj_boot_seek(int fd,off_t hi,off_t lo)4550 kobj_boot_seek(int fd, off_t hi, off_t lo)
4551 {
4552 #if defined(_OBP)
4553 return (BOP_SEEK(fd, lo) == -1 ? -1 : 0);
4554 #else
4555 return (BRD_SEEK(bfs_ops, fd, lo, SEEK_SET));
4556 #endif
4557 }
4558
4559 static int
kobj_boot_read(int fd,caddr_t buf,size_t size)4560 kobj_boot_read(int fd, caddr_t buf, size_t size)
4561 {
4562 #if defined(_OBP)
4563 return (BOP_READ(fd, buf, size));
4564 #else
4565 return (BRD_READ(bfs_ops, fd, buf, size));
4566 #endif
4567 }
4568
4569 static int
kobj_boot_compinfo(int fd,struct compinfo * cb)4570 kobj_boot_compinfo(int fd, struct compinfo *cb)
4571 {
4572 return (boot_compinfo(fd, cb));
4573 }
4574
4575 /*
4576 * Check if the file is compressed (for now we handle only gzip).
4577 * It returns CH_MAGIC_GZIP if the file is compressed and 0 otherwise.
4578 */
4579 static int
kobj_is_compressed(intptr_t fd)4580 kobj_is_compressed(intptr_t fd)
4581 {
4582 struct vnode *vp = (struct vnode *)fd;
4583 ssize_t resid;
4584 uint16_t magic_buf;
4585 int err = 0;
4586
4587 if ((err = vn_rdwr(UIO_READ, vp, (caddr_t)((intptr_t)&magic_buf),
4588 sizeof (magic_buf), (offset_t)(0),
4589 UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
4590
4591 _kobj_printf(ops, "kobj_is_compressed: vn_rdwr() failed, "
4592 "error code 0x%x\n", err);
4593 return (0);
4594 }
4595
4596 if (magic_buf == CH_MAGIC_GZIP)
4597 return (CH_MAGIC_GZIP);
4598
4599 return (0);
4600 }
4601