1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997-2000 Doug Rabson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include "opt_ddb.h"
31 #include "opt_kld.h"
32 #include "opt_hwpmc_hooks.h"
33 #include "opt_hwt_hooks.h"
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/boottrace.h>
38 #include <sys/eventhandler.h>
39 #include <sys/fcntl.h>
40 #include <sys/jail.h>
41 #include <sys/kernel.h>
42 #include <sys/libkern.h>
43 #include <sys/linker.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/module.h>
47 #include <sys/mount.h>
48 #include <sys/mutex.h>
49 #include <sys/namei.h>
50 #include <sys/priv.h>
51 #include <sys/proc.h>
52 #include <sys/sx.h>
53 #include <sys/syscallsubr.h>
54 #include <sys/sysctl.h>
55 #include <sys/sysproto.h>
56 #include <sys/vnode.h>
57
58 #ifdef DDB
59 #include <ddb/ddb.h>
60 #endif
61
62 #include <net/vnet.h>
63
64 #include <security/mac/mac_framework.h>
65
66 #include "linker_if.h"
67
68 #if defined(HWPMC_HOOKS) || defined(HWT_HOOKS)
69 #include <sys/pmckern.h>
70 #endif
71
72 #ifdef KLD_DEBUG
73 int kld_debug = 0;
74 SYSCTL_INT(_debug, OID_AUTO, kld_debug, CTLFLAG_RWTUN,
75 &kld_debug, 0, "Set various levels of KLD debug");
76 #endif
77
78 /* These variables are used by kernel debuggers to enumerate loaded files. */
79 const int kld_off_address = offsetof(struct linker_file, address);
80 const int kld_off_filename = offsetof(struct linker_file, filename);
81 const int kld_off_pathname = offsetof(struct linker_file, pathname);
82 const int kld_off_next = offsetof(struct linker_file, link.tqe_next);
83
84 /*
85 * static char *linker_search_path(const char *name, struct mod_depend
86 * *verinfo);
87 */
88 static const char *linker_basename(const char *path);
89
90 /*
91 * Find a currently loaded file given its filename.
92 */
93 static linker_file_t linker_find_file_by_name(const char* _filename);
94
95 /*
96 * Find a currently loaded file given its file id.
97 */
98 static linker_file_t linker_find_file_by_id(int _fileid);
99
100 /* Metadata from the static kernel */
101 SET_DECLARE(modmetadata_set, struct mod_metadata);
102
103 MALLOC_DEFINE(M_LINKER, "linker", "kernel linker");
104
105 linker_file_t linker_kernel_file;
106
107 static struct sx kld_sx; /* kernel linker lock */
108 static u_int kld_busy;
109 static struct thread *kld_busy_owner;
110
111 /*
112 * Load counter used by clients to determine if a linker file has been
113 * re-loaded. This counter is incremented for each file load.
114 */
115 static int loadcnt;
116
117 static linker_class_list_t classes;
118 static linker_file_list_t linker_files;
119 static int next_file_id = 1;
120 static int linker_no_more_classes = 0;
121
122 #define LINKER_GET_NEXT_FILE_ID(a) do { \
123 linker_file_t lftmp; \
124 \
125 if (!cold) \
126 sx_assert(&kld_sx, SA_XLOCKED); \
127 retry: \
128 TAILQ_FOREACH(lftmp, &linker_files, link) { \
129 if (next_file_id == lftmp->id) { \
130 next_file_id++; \
131 goto retry; \
132 } \
133 } \
134 (a) = next_file_id; \
135 } while (0)
136
137 /* XXX wrong name; we're looking at version provision tags here, not modules */
138 typedef TAILQ_HEAD(, modlist) modlisthead_t;
139 struct modlist {
140 TAILQ_ENTRY(modlist) link; /* chain together all modules */
141 linker_file_t container;
142 const char *name;
143 int version;
144 };
145 typedef struct modlist *modlist_t;
146 static modlisthead_t found_modules;
147
148 static void linker_file_add_dependency(linker_file_t file,
149 linker_file_t dep);
150 static caddr_t linker_file_lookup_symbol_internal(linker_file_t file,
151 const char* name, int deps);
152 static int linker_load_module(const char *kldname,
153 const char *modname, struct linker_file *parent,
154 const struct mod_depend *verinfo, struct linker_file **lfpp);
155 static modlist_t modlist_lookup2(const char *name, const struct mod_depend *verinfo);
156
157 static void
linker_init(void * arg)158 linker_init(void *arg)
159 {
160
161 sx_init(&kld_sx, "kernel linker");
162 TAILQ_INIT(&classes);
163 TAILQ_INIT(&linker_files);
164 }
165
166 SYSINIT(linker, SI_SUB_KLD, SI_ORDER_FIRST, linker_init, NULL);
167
168 static void
linker_stop_class_add(void * arg)169 linker_stop_class_add(void *arg)
170 {
171
172 linker_no_more_classes = 1;
173 }
174
175 SYSINIT(linker_class, SI_SUB_KLD, SI_ORDER_ANY, linker_stop_class_add, NULL);
176
177 int
linker_add_class(linker_class_t lc)178 linker_add_class(linker_class_t lc)
179 {
180
181 /*
182 * We disallow any class registration past SI_ORDER_ANY
183 * of SI_SUB_KLD. We bump the reference count to keep the
184 * ops from being freed.
185 */
186 if (linker_no_more_classes == 1)
187 return (EPERM);
188 kobj_class_compile((kobj_class_t) lc);
189 ((kobj_class_t)lc)->refs++; /* XXX: kobj_mtx */
190 TAILQ_INSERT_TAIL(&classes, lc, link);
191 return (0);
192 }
193
194 static void
linker_file_sysinit(linker_file_t lf)195 linker_file_sysinit(linker_file_t lf)
196 {
197 struct sysinit **start, **stop, **sipp, **xipp, *save;
198 int last;
199
200 KLD_DPF(FILE, ("linker_file_sysinit: calling SYSINITs for %s\n",
201 lf->filename));
202
203 sx_assert(&kld_sx, SA_XLOCKED);
204
205 if (linker_file_lookup_set(lf, "sysinit_set", &start, &stop, NULL) != 0)
206 return;
207 /*
208 * Perform a bubble sort of the system initialization objects by
209 * their subsystem (primary key) and order (secondary key).
210 *
211 * Since some things care about execution order, this is the operation
212 * which ensures continued function.
213 */
214 for (sipp = start; sipp < stop; sipp++) {
215 for (xipp = sipp + 1; xipp < stop; xipp++) {
216 if ((*sipp)->subsystem < (*xipp)->subsystem ||
217 ((*sipp)->subsystem == (*xipp)->subsystem &&
218 (*sipp)->order <= (*xipp)->order))
219 continue; /* skip */
220 save = *sipp;
221 *sipp = *xipp;
222 *xipp = save;
223 }
224 }
225
226 /*
227 * Traverse the (now) ordered list of system initialization tasks.
228 * Perform each task, and continue on to the next task.
229 */
230 last = SI_SUB_DUMMY;
231 sx_xunlock(&kld_sx);
232 mtx_lock(&Giant);
233 for (sipp = start; sipp < stop; sipp++) {
234 if ((*sipp)->subsystem == SI_SUB_DUMMY)
235 continue; /* skip dummy task(s) */
236
237 if ((*sipp)->subsystem > last)
238 BOOTTRACE("%s: sysinit 0x%7x", lf->filename,
239 (*sipp)->subsystem);
240
241 /* Call function */
242 (*((*sipp)->func)) ((*sipp)->udata);
243 last = (*sipp)->subsystem;
244 }
245 mtx_unlock(&Giant);
246 sx_xlock(&kld_sx);
247 }
248
249 static void
linker_file_sysuninit(linker_file_t lf)250 linker_file_sysuninit(linker_file_t lf)
251 {
252 struct sysinit **start, **stop, **sipp, **xipp, *save;
253 int last;
254
255 KLD_DPF(FILE, ("linker_file_sysuninit: calling SYSUNINITs for %s\n",
256 lf->filename));
257
258 sx_assert(&kld_sx, SA_XLOCKED);
259
260 if (linker_file_lookup_set(lf, "sysuninit_set", &start, &stop,
261 NULL) != 0)
262 return;
263
264 /*
265 * Perform a reverse bubble sort of the system initialization objects
266 * by their subsystem (primary key) and order (secondary key).
267 *
268 * Since some things care about execution order, this is the operation
269 * which ensures continued function.
270 */
271 for (sipp = start; sipp < stop; sipp++) {
272 for (xipp = sipp + 1; xipp < stop; xipp++) {
273 if ((*sipp)->subsystem > (*xipp)->subsystem ||
274 ((*sipp)->subsystem == (*xipp)->subsystem &&
275 (*sipp)->order >= (*xipp)->order))
276 continue; /* skip */
277 save = *sipp;
278 *sipp = *xipp;
279 *xipp = save;
280 }
281 }
282
283 /*
284 * Traverse the (now) ordered list of system initialization tasks.
285 * Perform each task, and continue on to the next task.
286 */
287 sx_xunlock(&kld_sx);
288 mtx_lock(&Giant);
289 last = SI_SUB_DUMMY;
290 for (sipp = start; sipp < stop; sipp++) {
291 if ((*sipp)->subsystem == SI_SUB_DUMMY)
292 continue; /* skip dummy task(s) */
293
294 if ((*sipp)->subsystem > last)
295 BOOTTRACE("%s: sysuninit 0x%7x", lf->filename,
296 (*sipp)->subsystem);
297
298 /* Call function */
299 (*((*sipp)->func)) ((*sipp)->udata);
300 last = (*sipp)->subsystem;
301 }
302 mtx_unlock(&Giant);
303 sx_xlock(&kld_sx);
304 }
305
306 static void
linker_file_register_sysctls(linker_file_t lf,bool enable)307 linker_file_register_sysctls(linker_file_t lf, bool enable)
308 {
309 struct sysctl_oid **start, **stop, **oidp;
310
311 KLD_DPF(FILE,
312 ("linker_file_register_sysctls: registering SYSCTLs for %s\n",
313 lf->filename));
314
315 sx_assert(&kld_sx, SA_XLOCKED);
316
317 if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
318 return;
319
320 sx_xunlock(&kld_sx);
321 sysctl_wlock();
322 for (oidp = start; oidp < stop; oidp++) {
323 if (enable)
324 sysctl_register_oid(*oidp);
325 else
326 sysctl_register_disabled_oid(*oidp);
327 }
328 sysctl_wunlock();
329 sx_xlock(&kld_sx);
330 }
331
332 /*
333 * Invoke the LINKER_CTF_GET implementation for this file. Existing
334 * implementations will load CTF info from the filesystem upon the first call
335 * and cache it in the kernel thereafter.
336 */
337 static void
linker_ctf_load_file(linker_file_t file)338 linker_ctf_load_file(linker_file_t file)
339 {
340 linker_ctf_t lc;
341 int error;
342
343 error = linker_ctf_get(file, &lc);
344 if (error == 0)
345 return;
346 if (bootverbose) {
347 printf("failed to load CTF for %s: %d\n", file->filename,
348 error);
349 }
350 }
351
352 static void
linker_file_enable_sysctls(linker_file_t lf)353 linker_file_enable_sysctls(linker_file_t lf)
354 {
355 struct sysctl_oid **start, **stop, **oidp;
356
357 KLD_DPF(FILE,
358 ("linker_file_enable_sysctls: enable SYSCTLs for %s\n",
359 lf->filename));
360
361 sx_assert(&kld_sx, SA_XLOCKED);
362
363 if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
364 return;
365
366 sx_xunlock(&kld_sx);
367 sysctl_wlock();
368 for (oidp = start; oidp < stop; oidp++)
369 sysctl_enable_oid(*oidp);
370 sysctl_wunlock();
371 sx_xlock(&kld_sx);
372 }
373
374 static void
linker_file_unregister_sysctls(linker_file_t lf)375 linker_file_unregister_sysctls(linker_file_t lf)
376 {
377 struct sysctl_oid **start, **stop, **oidp;
378
379 KLD_DPF(FILE, ("linker_file_unregister_sysctls: unregistering SYSCTLs"
380 " for %s\n", lf->filename));
381
382 sx_assert(&kld_sx, SA_XLOCKED);
383
384 if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
385 return;
386
387 sx_xunlock(&kld_sx);
388 sysctl_wlock();
389 for (oidp = start; oidp < stop; oidp++)
390 sysctl_unregister_oid(*oidp);
391 sysctl_wunlock();
392 sx_xlock(&kld_sx);
393 }
394
395 static int
linker_file_register_modules(linker_file_t lf)396 linker_file_register_modules(linker_file_t lf)
397 {
398 struct mod_metadata **start, **stop, **mdp;
399 const moduledata_t *moddata;
400 int first_error, error;
401
402 KLD_DPF(FILE, ("linker_file_register_modules: registering modules"
403 " in %s\n", lf->filename));
404
405 sx_assert(&kld_sx, SA_XLOCKED);
406
407 if (linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop, NULL) != 0) {
408 /*
409 * This fallback should be unnecessary, but if we get booted
410 * from boot2 instead of loader and we are missing our
411 * metadata then we have to try the best we can.
412 */
413 if (lf == linker_kernel_file) {
414 start = SET_BEGIN(modmetadata_set);
415 stop = SET_LIMIT(modmetadata_set);
416 } else
417 return (0);
418 }
419 first_error = 0;
420 for (mdp = start; mdp < stop; mdp++) {
421 if ((*mdp)->md_type != MDT_MODULE)
422 continue;
423 moddata = (*mdp)->md_data;
424 KLD_DPF(FILE, ("Registering module %s in %s\n",
425 moddata->name, lf->filename));
426 error = module_register(moddata, lf);
427 if (error) {
428 printf("Module %s failed to register: %d\n",
429 moddata->name, error);
430 if (first_error == 0)
431 first_error = error;
432 }
433 }
434 return (first_error);
435 }
436
437 static void
linker_init_kernel_modules(void * dummy __unused)438 linker_init_kernel_modules(void *dummy __unused)
439 {
440
441 sx_xlock(&kld_sx);
442 linker_file_register_modules(linker_kernel_file);
443 sx_xunlock(&kld_sx);
444 }
445
446 SYSINIT(linker_kernel, SI_SUB_KLD, SI_ORDER_ANY, linker_init_kernel_modules,
447 NULL);
448
449 static int
linker_load_file(const char * filename,linker_file_t * result)450 linker_load_file(const char *filename, linker_file_t *result)
451 {
452 linker_class_t lc;
453 linker_file_t lf;
454 int foundfile, error, modules;
455
456 /* Refuse to load modules if securelevel raised */
457 if (prison0.pr_securelevel > 0)
458 return (EPERM);
459
460 sx_assert(&kld_sx, SA_XLOCKED);
461 lf = linker_find_file_by_name(filename);
462 if (lf) {
463 KLD_DPF(FILE, ("linker_load_file: file %s is already loaded,"
464 " incrementing refs\n", filename));
465 *result = lf;
466 lf->refs++;
467 return (0);
468 }
469 foundfile = 0;
470 error = 0;
471
472 /*
473 * We do not need to protect (lock) classes here because there is
474 * no class registration past startup (SI_SUB_KLD, SI_ORDER_ANY)
475 * and there is no class deregistration mechanism at this time.
476 */
477 TAILQ_FOREACH(lc, &classes, link) {
478 KLD_DPF(FILE, ("linker_load_file: trying to load %s\n",
479 filename));
480 error = LINKER_LOAD_FILE(lc, filename, &lf);
481 /*
482 * If we got something other than ENOENT, then it exists but
483 * we cannot load it for some other reason.
484 */
485 if (error != ENOENT) {
486 foundfile = 1;
487 if (error == EEXIST)
488 break;
489 }
490 if (lf) {
491 error = linker_file_register_modules(lf);
492 if (error == EEXIST) {
493 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
494 return (error);
495 }
496 modules = !TAILQ_EMPTY(&lf->modules);
497 linker_file_register_sysctls(lf, false);
498 #ifdef VIMAGE
499 LINKER_PROPAGATE_VNETS(lf);
500 #endif
501 linker_file_sysinit(lf);
502 lf->flags |= LINKER_FILE_LINKED;
503
504 /*
505 * If all of the modules in this file failed
506 * to load, unload the file and return an
507 * error of ENOEXEC.
508 */
509 if (modules && TAILQ_EMPTY(&lf->modules)) {
510 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
511 return (ENOEXEC);
512 }
513 linker_file_enable_sysctls(lf);
514
515 /*
516 * Ask the linker to load CTF data for this file.
517 */
518 linker_ctf_load_file(lf);
519 EVENTHANDLER_INVOKE(kld_load, lf);
520 *result = lf;
521 return (0);
522 }
523 }
524 /*
525 * Less than ideal, but tells the user whether it failed to load or
526 * the module was not found.
527 */
528 if (foundfile) {
529 /*
530 * If the file type has not been recognized by the last try
531 * printout a message before to fail.
532 */
533 if (error == ENOSYS)
534 printf("%s: %s - unsupported file type\n",
535 __func__, filename);
536
537 /*
538 * Format not recognized or otherwise unloadable.
539 * When loading a module that is statically built into
540 * the kernel EEXIST percolates back up as the return
541 * value. Preserve this so that apps like sysinstall
542 * can recognize this special case and not post bogus
543 * dialog boxes.
544 */
545 if (error != EEXIST)
546 error = ENOEXEC;
547 } else
548 error = ENOENT; /* Nothing found */
549 return (error);
550 }
551
552 int
linker_reference_module(const char * modname,struct mod_depend * verinfo,linker_file_t * result)553 linker_reference_module(const char *modname, struct mod_depend *verinfo,
554 linker_file_t *result)
555 {
556 modlist_t mod;
557 int error;
558
559 sx_xlock(&kld_sx);
560 if ((mod = modlist_lookup2(modname, verinfo)) != NULL) {
561 *result = mod->container;
562 (*result)->refs++;
563 sx_xunlock(&kld_sx);
564 return (0);
565 }
566
567 error = linker_load_module(NULL, modname, NULL, verinfo, result);
568 sx_xunlock(&kld_sx);
569 return (error);
570 }
571
572 int
linker_release_module(const char * modname,struct mod_depend * verinfo,linker_file_t lf)573 linker_release_module(const char *modname, struct mod_depend *verinfo,
574 linker_file_t lf)
575 {
576 modlist_t mod;
577 int error;
578
579 sx_xlock(&kld_sx);
580 if (lf == NULL) {
581 KASSERT(modname != NULL,
582 ("linker_release_module: no file or name"));
583 mod = modlist_lookup2(modname, verinfo);
584 if (mod == NULL) {
585 sx_xunlock(&kld_sx);
586 return (ESRCH);
587 }
588 lf = mod->container;
589 } else
590 KASSERT(modname == NULL && verinfo == NULL,
591 ("linker_release_module: both file and name"));
592 error = linker_file_unload(lf, LINKER_UNLOAD_NORMAL);
593 sx_xunlock(&kld_sx);
594 return (error);
595 }
596
597 static linker_file_t
linker_find_file_by_name(const char * filename)598 linker_find_file_by_name(const char *filename)
599 {
600 linker_file_t lf;
601 char *koname;
602
603 koname = malloc(strlen(filename) + 4, M_LINKER, M_WAITOK);
604 sprintf(koname, "%s.ko", filename);
605
606 sx_assert(&kld_sx, SA_XLOCKED);
607 TAILQ_FOREACH(lf, &linker_files, link) {
608 if (strcmp(lf->filename, koname) == 0)
609 break;
610 if (strcmp(lf->filename, filename) == 0)
611 break;
612 }
613 free(koname, M_LINKER);
614 return (lf);
615 }
616
617 static linker_file_t
linker_find_file_by_id(int fileid)618 linker_find_file_by_id(int fileid)
619 {
620 linker_file_t lf;
621
622 sx_assert(&kld_sx, SA_XLOCKED);
623 TAILQ_FOREACH(lf, &linker_files, link)
624 if (lf->id == fileid && lf->flags & LINKER_FILE_LINKED)
625 break;
626 return (lf);
627 }
628
629 int
linker_file_foreach(linker_predicate_t * predicate,void * context)630 linker_file_foreach(linker_predicate_t *predicate, void *context)
631 {
632 linker_file_t lf;
633 int retval = 0;
634
635 sx_xlock(&kld_sx);
636 TAILQ_FOREACH(lf, &linker_files, link) {
637 retval = predicate(lf, context);
638 if (retval != 0)
639 break;
640 }
641 sx_xunlock(&kld_sx);
642 return (retval);
643 }
644
645 linker_file_t
linker_make_file(const char * pathname,linker_class_t lc)646 linker_make_file(const char *pathname, linker_class_t lc)
647 {
648 linker_file_t lf;
649 const char *filename;
650
651 if (!cold)
652 sx_assert(&kld_sx, SA_XLOCKED);
653 filename = linker_basename(pathname);
654
655 KLD_DPF(FILE, ("linker_make_file: new file, filename='%s' for pathname='%s'\n", filename, pathname));
656 lf = (linker_file_t)kobj_create((kobj_class_t)lc, M_LINKER, M_WAITOK);
657 if (lf == NULL)
658 return (NULL);
659 lf->ctors_addr = 0;
660 lf->ctors_size = 0;
661 lf->ctors_invoked = LF_NONE;
662 lf->dtors_addr = 0;
663 lf->dtors_size = 0;
664 lf->refs = 1;
665 lf->userrefs = 0;
666 lf->flags = 0;
667 lf->filename = strdup(filename, M_LINKER);
668 lf->pathname = strdup(pathname, M_LINKER);
669 LINKER_GET_NEXT_FILE_ID(lf->id);
670 lf->ndeps = 0;
671 lf->deps = NULL;
672 lf->loadcnt = ++loadcnt;
673 #ifdef __arm__
674 lf->exidx_addr = 0;
675 lf->exidx_size = 0;
676 #endif
677 STAILQ_INIT(&lf->common);
678 TAILQ_INIT(&lf->modules);
679 TAILQ_INSERT_TAIL(&linker_files, lf, link);
680 return (lf);
681 }
682
683 int
linker_file_unload(linker_file_t file,int flags)684 linker_file_unload(linker_file_t file, int flags)
685 {
686 module_t mod, next;
687 modlist_t ml, nextml;
688 struct common_symbol *cp;
689 int error, i;
690
691 /* Refuse to unload modules if securelevel raised. */
692 if (prison0.pr_securelevel > 0)
693 return (EPERM);
694
695 sx_assert(&kld_sx, SA_XLOCKED);
696 KLD_DPF(FILE, ("linker_file_unload: lf->refs=%d\n", file->refs));
697
698 /* Easy case of just dropping a reference. */
699 if (file->refs > 1) {
700 file->refs--;
701 return (0);
702 }
703
704 /* Give eventhandlers a chance to prevent the unload. */
705 error = 0;
706 if ((file->flags & LINKER_FILE_LINKED) != 0) {
707 EVENTHANDLER_INVOKE(kld_unload_try, file, &error);
708 if (error != 0)
709 return (EBUSY);
710 }
711
712 KLD_DPF(FILE, ("linker_file_unload: file is unloading,"
713 " informing modules\n"));
714
715 /*
716 * Quiesce all the modules to give them a chance to veto the unload.
717 */
718 MOD_SLOCK;
719 for (mod = TAILQ_FIRST(&file->modules); mod;
720 mod = module_getfnext(mod)) {
721 error = module_quiesce(mod);
722 if (error != 0 && flags != LINKER_UNLOAD_FORCE) {
723 KLD_DPF(FILE, ("linker_file_unload: module %s"
724 " vetoed unload\n", module_getname(mod)));
725 /*
726 * XXX: Do we need to tell all the quiesced modules
727 * that they can resume work now via a new module
728 * event?
729 */
730 MOD_SUNLOCK;
731 return (error);
732 }
733 }
734 MOD_SUNLOCK;
735
736 /*
737 * Inform any modules associated with this file that they are
738 * being unloaded.
739 */
740 MOD_XLOCK;
741 for (mod = TAILQ_FIRST(&file->modules); mod; mod = next) {
742 next = module_getfnext(mod);
743 MOD_XUNLOCK;
744
745 /*
746 * Give the module a chance to veto the unload.
747 */
748 if ((error = module_unload(mod)) != 0) {
749 #ifdef KLD_DEBUG
750 MOD_SLOCK;
751 KLD_DPF(FILE, ("linker_file_unload: module %s"
752 " failed unload\n", module_getname(mod)));
753 MOD_SUNLOCK;
754 #endif
755 return (error);
756 }
757 MOD_XLOCK;
758 module_release(mod);
759 }
760 MOD_XUNLOCK;
761
762 TAILQ_FOREACH_SAFE(ml, &found_modules, link, nextml) {
763 if (ml->container == file) {
764 TAILQ_REMOVE(&found_modules, ml, link);
765 free(ml, M_LINKER);
766 }
767 }
768
769 /*
770 * Don't try to run SYSUNINITs if we are unloaded due to a
771 * link error.
772 */
773 if ((file->flags & LINKER_FILE_LINKED) != 0) {
774 file->flags &= ~LINKER_FILE_LINKED;
775 linker_file_unregister_sysctls(file);
776 linker_file_sysuninit(file);
777 EVENTHANDLER_INVOKE(kld_unload, file->filename, file->address,
778 file->size);
779 }
780 TAILQ_REMOVE(&linker_files, file, link);
781
782 if (file->deps) {
783 for (i = 0; i < file->ndeps; i++)
784 linker_file_unload(file->deps[i], flags);
785 free(file->deps, M_LINKER);
786 file->deps = NULL;
787 }
788 while ((cp = STAILQ_FIRST(&file->common)) != NULL) {
789 STAILQ_REMOVE_HEAD(&file->common, link);
790 free(cp, M_LINKER);
791 }
792
793 LINKER_UNLOAD(file);
794
795 if (file->filename) {
796 free(file->filename, M_LINKER);
797 file->filename = NULL;
798 }
799 if (file->pathname) {
800 free(file->pathname, M_LINKER);
801 file->pathname = NULL;
802 }
803 kobj_delete((kobj_t) file, M_LINKER);
804 return (0);
805 }
806
807 int
linker_ctf_get(linker_file_t file,linker_ctf_t * lc)808 linker_ctf_get(linker_file_t file, linker_ctf_t *lc)
809 {
810 return (LINKER_CTF_GET(file, lc));
811 }
812
813 int
linker_ctf_lookup_typename_ddb(linker_ctf_t * lc,const char * typename)814 linker_ctf_lookup_typename_ddb(linker_ctf_t *lc, const char *typename)
815 {
816 #ifdef DDB
817 linker_file_t lf;
818
819 TAILQ_FOREACH (lf, &linker_files, link){
820 if (LINKER_CTF_LOOKUP_TYPENAME(lf, lc, typename) == 0)
821 return (0);
822 }
823 #endif
824 return (ENOENT);
825 }
826
827 int
linker_ctf_lookup_sym_ddb(const char * symname,c_linker_sym_t * sym,linker_ctf_t * lc)828 linker_ctf_lookup_sym_ddb(const char *symname, c_linker_sym_t *sym,
829 linker_ctf_t *lc)
830 {
831 #ifdef DDB
832 linker_file_t lf;
833
834 TAILQ_FOREACH (lf, &linker_files, link){
835 if (LINKER_LOOKUP_DEBUG_SYMBOL_CTF(lf, symname, sym, lc) == 0)
836 return (0);
837 }
838 #endif
839 return (ENOENT);
840 }
841
842 static void
linker_file_add_dependency(linker_file_t file,linker_file_t dep)843 linker_file_add_dependency(linker_file_t file, linker_file_t dep)
844 {
845 linker_file_t *newdeps;
846
847 sx_assert(&kld_sx, SA_XLOCKED);
848 file->deps = realloc(file->deps, (file->ndeps + 1) * sizeof(*newdeps),
849 M_LINKER, M_WAITOK | M_ZERO);
850 file->deps[file->ndeps] = dep;
851 file->ndeps++;
852 KLD_DPF(FILE, ("linker_file_add_dependency:"
853 " adding %s as dependency for %s\n",
854 dep->filename, file->filename));
855 }
856
857 /*
858 * Locate a linker set and its contents. This is a helper function to avoid
859 * linker_if.h exposure elsewhere. Note: firstp and lastp are really void **.
860 * This function is used in this file so we can avoid having lots of (void **)
861 * casts.
862 */
863 int
linker_file_lookup_set(linker_file_t file,const char * name,void * firstp,void * lastp,int * countp)864 linker_file_lookup_set(linker_file_t file, const char *name,
865 void *firstp, void *lastp, int *countp)
866 {
867
868 sx_assert(&kld_sx, SA_LOCKED);
869 return (LINKER_LOOKUP_SET(file, name, firstp, lastp, countp));
870 }
871
872 /*
873 * List all functions in a file.
874 */
875 int
linker_file_function_listall(linker_file_t lf,linker_function_nameval_callback_t callback_func,void * arg)876 linker_file_function_listall(linker_file_t lf,
877 linker_function_nameval_callback_t callback_func, void *arg)
878 {
879 return (LINKER_EACH_FUNCTION_NAMEVAL(lf, callback_func, arg));
880 }
881
882 caddr_t
linker_file_lookup_symbol(linker_file_t file,const char * name,int deps)883 linker_file_lookup_symbol(linker_file_t file, const char *name, int deps)
884 {
885 caddr_t sym;
886 int locked;
887
888 locked = sx_xlocked(&kld_sx);
889 if (!locked)
890 sx_xlock(&kld_sx);
891 sym = linker_file_lookup_symbol_internal(file, name, deps);
892 if (!locked)
893 sx_xunlock(&kld_sx);
894 return (sym);
895 }
896
897 static caddr_t
linker_file_lookup_symbol_internal(linker_file_t file,const char * name,int deps)898 linker_file_lookup_symbol_internal(linker_file_t file, const char *name,
899 int deps)
900 {
901 c_linker_sym_t sym;
902 linker_symval_t symval;
903 caddr_t address;
904 size_t common_size = 0;
905 int i;
906
907 sx_assert(&kld_sx, SA_XLOCKED);
908 KLD_DPF(SYM, ("linker_file_lookup_symbol: file=%p, name=%s, deps=%d\n",
909 file, name, deps));
910
911 /*
912 * Treat the __this_linker_file as a special symbol. This is a
913 * global that linuxkpi uses to populate the THIS_MODULE
914 * value. In this case we can simply return the linker_file_t.
915 *
916 * Modules compiled statically into the kernel are assigned NULL.
917 */
918 if (strcmp(name, "__this_linker_file") == 0) {
919 address = (file == linker_kernel_file) ? NULL : (caddr_t)file;
920 KLD_DPF(SYM, ("linker_file_lookup_symbol: resolving special "
921 "symbol __this_linker_file to %p\n", address));
922 return (address);
923 }
924
925 if (LINKER_LOOKUP_SYMBOL(file, name, &sym) == 0) {
926 LINKER_SYMBOL_VALUES(file, sym, &symval);
927 if (symval.value == 0)
928 /*
929 * For commons, first look them up in the
930 * dependencies and only allocate space if not found
931 * there.
932 */
933 common_size = symval.size;
934 else {
935 KLD_DPF(SYM, ("linker_file_lookup_symbol: symbol"
936 ".value=%p\n", symval.value));
937 return (symval.value);
938 }
939 }
940 if (deps) {
941 for (i = 0; i < file->ndeps; i++) {
942 address = linker_file_lookup_symbol_internal(
943 file->deps[i], name, 0);
944 if (address) {
945 KLD_DPF(SYM, ("linker_file_lookup_symbol:"
946 " deps value=%p\n", address));
947 return (address);
948 }
949 }
950 }
951 if (common_size > 0) {
952 /*
953 * This is a common symbol which was not found in the
954 * dependencies. We maintain a simple common symbol table in
955 * the file object.
956 */
957 struct common_symbol *cp;
958
959 STAILQ_FOREACH(cp, &file->common, link) {
960 if (strcmp(cp->name, name) == 0) {
961 KLD_DPF(SYM, ("linker_file_lookup_symbol:"
962 " old common value=%p\n", cp->address));
963 return (cp->address);
964 }
965 }
966 /*
967 * Round the symbol size up to align.
968 */
969 common_size = (common_size + sizeof(int) - 1) & -sizeof(int);
970 cp = malloc(sizeof(struct common_symbol)
971 + common_size + strlen(name) + 1, M_LINKER,
972 M_WAITOK | M_ZERO);
973 cp->address = (caddr_t)(cp + 1);
974 cp->name = cp->address + common_size;
975 strcpy(cp->name, name);
976 bzero(cp->address, common_size);
977 STAILQ_INSERT_TAIL(&file->common, cp, link);
978
979 KLD_DPF(SYM, ("linker_file_lookup_symbol: new common"
980 " value=%p\n", cp->address));
981 return (cp->address);
982 }
983 KLD_DPF(SYM, ("linker_file_lookup_symbol: fail\n"));
984 return (0);
985 }
986
987 /*
988 * Both DDB and stack(9) rely on the kernel linker to provide forward and
989 * backward lookup of symbols. However, DDB and sometimes stack(9) need to
990 * do this in a lockfree manner. We provide a set of internal helper
991 * routines to perform these operations without locks, and then wrappers that
992 * optionally lock.
993 *
994 * linker_debug_lookup() is ifdef DDB as currently it's only used by DDB.
995 */
996 #ifdef DDB
997 static int
linker_debug_lookup(const char * symstr,c_linker_sym_t * sym)998 linker_debug_lookup(const char *symstr, c_linker_sym_t *sym)
999 {
1000 linker_file_t lf;
1001
1002 TAILQ_FOREACH(lf, &linker_files, link) {
1003 if (LINKER_LOOKUP_DEBUG_SYMBOL(lf, symstr, sym) == 0)
1004 return (0);
1005 }
1006 return (ENOENT);
1007 }
1008 #endif
1009
1010 static int
linker_debug_search_symbol(caddr_t value,c_linker_sym_t * sym,long * diffp)1011 linker_debug_search_symbol(caddr_t value, c_linker_sym_t *sym, long *diffp)
1012 {
1013 linker_file_t lf;
1014 c_linker_sym_t best, es;
1015 u_long diff, bestdiff, off;
1016
1017 best = 0;
1018 off = (uintptr_t)value;
1019 bestdiff = off;
1020 TAILQ_FOREACH(lf, &linker_files, link) {
1021 if (LINKER_SEARCH_SYMBOL(lf, value, &es, &diff) != 0)
1022 continue;
1023 if (es != 0 && diff < bestdiff) {
1024 best = es;
1025 bestdiff = diff;
1026 }
1027 if (bestdiff == 0)
1028 break;
1029 }
1030 if (best) {
1031 *sym = best;
1032 *diffp = bestdiff;
1033 return (0);
1034 } else {
1035 *sym = 0;
1036 *diffp = off;
1037 return (ENOENT);
1038 }
1039 }
1040
1041 static int
linker_debug_symbol_values(c_linker_sym_t sym,linker_symval_t * symval)1042 linker_debug_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
1043 {
1044 linker_file_t lf;
1045
1046 TAILQ_FOREACH(lf, &linker_files, link) {
1047 if (LINKER_DEBUG_SYMBOL_VALUES(lf, sym, symval) == 0)
1048 return (0);
1049 }
1050 return (ENOENT);
1051 }
1052
1053 static int
linker_debug_search_symbol_name(caddr_t value,char * buf,u_int buflen,long * offset)1054 linker_debug_search_symbol_name(caddr_t value, char *buf, u_int buflen,
1055 long *offset)
1056 {
1057 linker_symval_t symval;
1058 c_linker_sym_t sym;
1059 int error;
1060
1061 *offset = 0;
1062 error = linker_debug_search_symbol(value, &sym, offset);
1063 if (error)
1064 return (error);
1065 error = linker_debug_symbol_values(sym, &symval);
1066 if (error)
1067 return (error);
1068 strlcpy(buf, symval.name, buflen);
1069 return (0);
1070 }
1071
1072 /*
1073 * DDB Helpers. DDB has to look across multiple files with their own symbol
1074 * tables and string tables.
1075 *
1076 * Note that we do not obey list locking protocols here. We really don't need
1077 * DDB to hang because somebody's got the lock held. We'll take the chance
1078 * that the files list is inconsistent instead.
1079 */
1080 #ifdef DDB
1081 int
linker_ddb_lookup(const char * symstr,c_linker_sym_t * sym)1082 linker_ddb_lookup(const char *symstr, c_linker_sym_t *sym)
1083 {
1084
1085 return (linker_debug_lookup(symstr, sym));
1086 }
1087 #endif
1088
1089 int
linker_ddb_search_symbol(caddr_t value,c_linker_sym_t * sym,long * diffp)1090 linker_ddb_search_symbol(caddr_t value, c_linker_sym_t *sym, long *diffp)
1091 {
1092
1093 return (linker_debug_search_symbol(value, sym, diffp));
1094 }
1095
1096 int
linker_ddb_symbol_values(c_linker_sym_t sym,linker_symval_t * symval)1097 linker_ddb_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
1098 {
1099
1100 return (linker_debug_symbol_values(sym, symval));
1101 }
1102
1103 int
linker_ddb_search_symbol_name(caddr_t value,char * buf,u_int buflen,long * offset)1104 linker_ddb_search_symbol_name(caddr_t value, char *buf, u_int buflen,
1105 long *offset)
1106 {
1107
1108 return (linker_debug_search_symbol_name(value, buf, buflen, offset));
1109 }
1110
1111 /*
1112 * stack(9) helper for non-debugging environemnts. Unlike DDB helpers, we do
1113 * obey locking protocols, and offer a significantly less complex interface.
1114 */
1115 int
linker_search_symbol_name_flags(caddr_t value,char * buf,u_int buflen,long * offset,int flags)1116 linker_search_symbol_name_flags(caddr_t value, char *buf, u_int buflen,
1117 long *offset, int flags)
1118 {
1119 int error;
1120
1121 KASSERT((flags & (M_NOWAIT | M_WAITOK)) != 0 &&
1122 (flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK),
1123 ("%s: bad flags: 0x%x", __func__, flags));
1124
1125 if (flags & M_NOWAIT) {
1126 if (!sx_try_slock(&kld_sx))
1127 return (EWOULDBLOCK);
1128 } else
1129 sx_slock(&kld_sx);
1130
1131 error = linker_debug_search_symbol_name(value, buf, buflen, offset);
1132 sx_sunlock(&kld_sx);
1133 return (error);
1134 }
1135
1136 int
linker_search_symbol_name(caddr_t value,char * buf,u_int buflen,long * offset)1137 linker_search_symbol_name(caddr_t value, char *buf, u_int buflen,
1138 long *offset)
1139 {
1140
1141 return (linker_search_symbol_name_flags(value, buf, buflen, offset,
1142 M_WAITOK));
1143 }
1144
1145 int
linker_kldload_busy(int flags)1146 linker_kldload_busy(int flags)
1147 {
1148 int error;
1149
1150 MPASS((flags & ~(LINKER_UB_UNLOCK | LINKER_UB_LOCKED |
1151 LINKER_UB_PCATCH)) == 0);
1152 if ((flags & LINKER_UB_LOCKED) != 0)
1153 sx_assert(&kld_sx, SA_XLOCKED);
1154
1155 if ((flags & LINKER_UB_LOCKED) == 0)
1156 sx_xlock(&kld_sx);
1157 while (kld_busy > 0) {
1158 if (kld_busy_owner == curthread)
1159 break;
1160 error = sx_sleep(&kld_busy, &kld_sx,
1161 (flags & LINKER_UB_PCATCH) != 0 ? PCATCH : 0,
1162 "kldbusy", 0);
1163 if (error != 0) {
1164 if ((flags & LINKER_UB_UNLOCK) != 0)
1165 sx_xunlock(&kld_sx);
1166 return (error);
1167 }
1168 }
1169 kld_busy++;
1170 kld_busy_owner = curthread;
1171 if ((flags & LINKER_UB_UNLOCK) != 0)
1172 sx_xunlock(&kld_sx);
1173 return (0);
1174 }
1175
1176 void
linker_kldload_unbusy(int flags)1177 linker_kldload_unbusy(int flags)
1178 {
1179 MPASS((flags & ~LINKER_UB_LOCKED) == 0);
1180 if ((flags & LINKER_UB_LOCKED) != 0)
1181 sx_assert(&kld_sx, SA_XLOCKED);
1182
1183 if ((flags & LINKER_UB_LOCKED) == 0)
1184 sx_xlock(&kld_sx);
1185 MPASS(kld_busy > 0);
1186 if (kld_busy_owner != curthread)
1187 panic("linker_kldload_unbusy done by not owning thread %p",
1188 kld_busy_owner);
1189 kld_busy--;
1190 if (kld_busy == 0) {
1191 kld_busy_owner = NULL;
1192 wakeup(&kld_busy);
1193 }
1194 sx_xunlock(&kld_sx);
1195 }
1196
1197 /*
1198 * Syscalls.
1199 */
1200 int
kern_kldload(struct thread * td,const char * file,int * fileid)1201 kern_kldload(struct thread *td, const char *file, int *fileid)
1202 {
1203 const char *kldname, *modname;
1204 linker_file_t lf;
1205 int error;
1206
1207 if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
1208 return (error);
1209
1210 if ((error = priv_check(td, PRIV_KLD_LOAD)) != 0)
1211 return (error);
1212
1213 /*
1214 * If file does not contain a qualified name or any dot in it
1215 * (kldname.ko, or kldname.ver.ko) treat it as an interface
1216 * name.
1217 */
1218 if (strchr(file, '/') || strchr(file, '.')) {
1219 kldname = file;
1220 modname = NULL;
1221 } else {
1222 kldname = NULL;
1223 modname = file;
1224 }
1225
1226 error = linker_kldload_busy(LINKER_UB_PCATCH);
1227 if (error != 0) {
1228 sx_xunlock(&kld_sx);
1229 return (error);
1230 }
1231
1232 /*
1233 * It is possible that kldloaded module will attach a new ifnet,
1234 * so vnet context must be set when this ocurs.
1235 */
1236 CURVNET_SET(TD_TO_VNET(td));
1237
1238 error = linker_load_module(kldname, modname, NULL, NULL, &lf);
1239 CURVNET_RESTORE();
1240
1241 if (error == 0) {
1242 lf->userrefs++;
1243 if (fileid != NULL)
1244 *fileid = lf->id;
1245 }
1246 linker_kldload_unbusy(LINKER_UB_LOCKED);
1247 return (error);
1248 }
1249
1250 int
sys_kldload(struct thread * td,struct kldload_args * uap)1251 sys_kldload(struct thread *td, struct kldload_args *uap)
1252 {
1253 char *pathname = NULL;
1254 int error, fileid;
1255
1256 td->td_retval[0] = -1;
1257
1258 pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
1259 error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL);
1260 if (error == 0) {
1261 error = kern_kldload(td, pathname, &fileid);
1262 if (error == 0)
1263 td->td_retval[0] = fileid;
1264 }
1265 free(pathname, M_TEMP);
1266 return (error);
1267 }
1268
1269 int
kern_kldunload(struct thread * td,int fileid,int flags)1270 kern_kldunload(struct thread *td, int fileid, int flags)
1271 {
1272 linker_file_t lf;
1273 int error = 0;
1274
1275 if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
1276 return (error);
1277
1278 if ((error = priv_check(td, PRIV_KLD_UNLOAD)) != 0)
1279 return (error);
1280
1281 error = linker_kldload_busy(LINKER_UB_PCATCH);
1282 if (error != 0) {
1283 sx_xunlock(&kld_sx);
1284 return (error);
1285 }
1286
1287 CURVNET_SET(TD_TO_VNET(td));
1288 lf = linker_find_file_by_id(fileid);
1289 if (lf) {
1290 KLD_DPF(FILE, ("kldunload: lf->userrefs=%d\n", lf->userrefs));
1291
1292 if (lf->userrefs == 0) {
1293 /*
1294 * XXX: maybe LINKER_UNLOAD_FORCE should override ?
1295 */
1296 printf("kldunload: attempt to unload file that was"
1297 " loaded by the kernel\n");
1298 error = EBUSY;
1299 } else if (lf->refs > 1) {
1300 error = EBUSY;
1301 } else {
1302 lf->userrefs--;
1303 error = linker_file_unload(lf, flags);
1304 if (error)
1305 lf->userrefs++;
1306 }
1307 } else
1308 error = ENOENT;
1309 CURVNET_RESTORE();
1310 linker_kldload_unbusy(LINKER_UB_LOCKED);
1311 return (error);
1312 }
1313
1314 int
sys_kldunload(struct thread * td,struct kldunload_args * uap)1315 sys_kldunload(struct thread *td, struct kldunload_args *uap)
1316 {
1317
1318 return (kern_kldunload(td, uap->fileid, LINKER_UNLOAD_NORMAL));
1319 }
1320
1321 int
sys_kldunloadf(struct thread * td,struct kldunloadf_args * uap)1322 sys_kldunloadf(struct thread *td, struct kldunloadf_args *uap)
1323 {
1324
1325 if (uap->flags != LINKER_UNLOAD_NORMAL &&
1326 uap->flags != LINKER_UNLOAD_FORCE)
1327 return (EINVAL);
1328 return (kern_kldunload(td, uap->fileid, uap->flags));
1329 }
1330
1331 int
sys_kldfind(struct thread * td,struct kldfind_args * uap)1332 sys_kldfind(struct thread *td, struct kldfind_args *uap)
1333 {
1334 char *pathname;
1335 const char *filename;
1336 linker_file_t lf;
1337 int error;
1338
1339 #ifdef MAC
1340 error = mac_kld_check_stat(td->td_ucred);
1341 if (error)
1342 return (error);
1343 #endif
1344
1345 td->td_retval[0] = -1;
1346
1347 pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
1348 if ((error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL)) != 0)
1349 goto out;
1350
1351 filename = linker_basename(pathname);
1352 sx_xlock(&kld_sx);
1353 lf = linker_find_file_by_name(filename);
1354 if (lf)
1355 td->td_retval[0] = lf->id;
1356 else
1357 error = ENOENT;
1358 sx_xunlock(&kld_sx);
1359 out:
1360 free(pathname, M_TEMP);
1361 return (error);
1362 }
1363
1364 int
sys_kldnext(struct thread * td,struct kldnext_args * uap)1365 sys_kldnext(struct thread *td, struct kldnext_args *uap)
1366 {
1367 linker_file_t lf;
1368 int error = 0;
1369
1370 #ifdef MAC
1371 error = mac_kld_check_stat(td->td_ucred);
1372 if (error)
1373 return (error);
1374 #endif
1375
1376 sx_xlock(&kld_sx);
1377 if (uap->fileid == 0)
1378 lf = TAILQ_FIRST(&linker_files);
1379 else {
1380 lf = linker_find_file_by_id(uap->fileid);
1381 if (lf == NULL) {
1382 error = ENOENT;
1383 goto out;
1384 }
1385 lf = TAILQ_NEXT(lf, link);
1386 }
1387
1388 /* Skip partially loaded files. */
1389 while (lf != NULL && !(lf->flags & LINKER_FILE_LINKED))
1390 lf = TAILQ_NEXT(lf, link);
1391
1392 if (lf)
1393 td->td_retval[0] = lf->id;
1394 else
1395 td->td_retval[0] = 0;
1396 out:
1397 sx_xunlock(&kld_sx);
1398 return (error);
1399 }
1400
1401 int
sys_kldstat(struct thread * td,struct kldstat_args * uap)1402 sys_kldstat(struct thread *td, struct kldstat_args *uap)
1403 {
1404 struct kld_file_stat *stat;
1405 int error, version;
1406
1407 /*
1408 * Check the version of the user's structure.
1409 */
1410 if ((error = copyin(&uap->stat->version, &version, sizeof(version)))
1411 != 0)
1412 return (error);
1413 if (version != sizeof(struct kld_file_stat_1) &&
1414 version != sizeof(struct kld_file_stat))
1415 return (EINVAL);
1416
1417 stat = malloc(sizeof(*stat), M_TEMP, M_WAITOK | M_ZERO);
1418 error = kern_kldstat(td, uap->fileid, stat);
1419 if (error == 0)
1420 error = copyout(stat, uap->stat, version);
1421 free(stat, M_TEMP);
1422 return (error);
1423 }
1424
1425 int
kern_kldstat(struct thread * td,int fileid,struct kld_file_stat * stat)1426 kern_kldstat(struct thread *td, int fileid, struct kld_file_stat *stat)
1427 {
1428 linker_file_t lf;
1429 int namelen;
1430 #ifdef MAC
1431 int error;
1432
1433 error = mac_kld_check_stat(td->td_ucred);
1434 if (error)
1435 return (error);
1436 #endif
1437
1438 sx_xlock(&kld_sx);
1439 lf = linker_find_file_by_id(fileid);
1440 if (lf == NULL) {
1441 sx_xunlock(&kld_sx);
1442 return (ENOENT);
1443 }
1444
1445 /* Version 1 fields: */
1446 namelen = strlen(lf->filename) + 1;
1447 if (namelen > sizeof(stat->name))
1448 namelen = sizeof(stat->name);
1449 bcopy(lf->filename, &stat->name[0], namelen);
1450 stat->refs = lf->refs;
1451 stat->id = lf->id;
1452 stat->address = lf->address;
1453 stat->size = lf->size;
1454 /* Version 2 fields: */
1455 namelen = strlen(lf->pathname) + 1;
1456 if (namelen > sizeof(stat->pathname))
1457 namelen = sizeof(stat->pathname);
1458 bcopy(lf->pathname, &stat->pathname[0], namelen);
1459 sx_xunlock(&kld_sx);
1460
1461 td->td_retval[0] = 0;
1462 return (0);
1463 }
1464
1465 #ifdef DDB
DB_COMMAND_FLAGS(kldstat,db_kldstat,DB_CMD_MEMSAFE)1466 DB_COMMAND_FLAGS(kldstat, db_kldstat, DB_CMD_MEMSAFE)
1467 {
1468 linker_file_t lf;
1469
1470 #define POINTER_WIDTH ((int)(sizeof(void *) * 2 + 2))
1471 db_printf("Id Refs Address%*c Size Name\n", POINTER_WIDTH - 7, ' ');
1472 #undef POINTER_WIDTH
1473 TAILQ_FOREACH(lf, &linker_files, link) {
1474 if (db_pager_quit)
1475 return;
1476 db_printf("%2d %4d %p %-8zx %s\n", lf->id, lf->refs,
1477 lf->address, lf->size, lf->filename);
1478 }
1479 }
1480 #endif /* DDB */
1481
1482 int
sys_kldfirstmod(struct thread * td,struct kldfirstmod_args * uap)1483 sys_kldfirstmod(struct thread *td, struct kldfirstmod_args *uap)
1484 {
1485 linker_file_t lf;
1486 module_t mp;
1487 int error = 0;
1488
1489 #ifdef MAC
1490 error = mac_kld_check_stat(td->td_ucred);
1491 if (error)
1492 return (error);
1493 #endif
1494
1495 sx_xlock(&kld_sx);
1496 lf = linker_find_file_by_id(uap->fileid);
1497 if (lf) {
1498 MOD_SLOCK;
1499 mp = TAILQ_FIRST(&lf->modules);
1500 if (mp != NULL)
1501 td->td_retval[0] = module_getid(mp);
1502 else
1503 td->td_retval[0] = 0;
1504 MOD_SUNLOCK;
1505 } else
1506 error = ENOENT;
1507 sx_xunlock(&kld_sx);
1508 return (error);
1509 }
1510
1511 int
sys_kldsym(struct thread * td,struct kldsym_args * uap)1512 sys_kldsym(struct thread *td, struct kldsym_args *uap)
1513 {
1514 char *symstr = NULL;
1515 c_linker_sym_t sym;
1516 linker_symval_t symval;
1517 linker_file_t lf;
1518 struct kld_sym_lookup lookup;
1519 int error = 0;
1520
1521 #ifdef MAC
1522 error = mac_kld_check_stat(td->td_ucred);
1523 if (error)
1524 return (error);
1525 #endif
1526
1527 if ((error = copyin(uap->data, &lookup, sizeof(lookup))) != 0)
1528 return (error);
1529 if (lookup.version != sizeof(lookup) ||
1530 uap->cmd != KLDSYM_LOOKUP)
1531 return (EINVAL);
1532 symstr = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
1533 if ((error = copyinstr(lookup.symname, symstr, MAXPATHLEN, NULL)) != 0)
1534 goto out;
1535 sx_xlock(&kld_sx);
1536 if (uap->fileid != 0) {
1537 lf = linker_find_file_by_id(uap->fileid);
1538 if (lf == NULL)
1539 error = ENOENT;
1540 else if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 &&
1541 LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) {
1542 lookup.symvalue = (uintptr_t) symval.value;
1543 lookup.symsize = symval.size;
1544 error = copyout(&lookup, uap->data, sizeof(lookup));
1545 } else
1546 error = ENOENT;
1547 } else {
1548 TAILQ_FOREACH(lf, &linker_files, link) {
1549 if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 &&
1550 LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) {
1551 lookup.symvalue = (uintptr_t)symval.value;
1552 lookup.symsize = symval.size;
1553 error = copyout(&lookup, uap->data,
1554 sizeof(lookup));
1555 break;
1556 }
1557 }
1558 if (lf == NULL)
1559 error = ENOENT;
1560 }
1561 sx_xunlock(&kld_sx);
1562 out:
1563 free(symstr, M_TEMP);
1564 return (error);
1565 }
1566
1567 /*
1568 * Preloaded module support
1569 */
1570
1571 static modlist_t
modlist_lookup(const char * name,int ver)1572 modlist_lookup(const char *name, int ver)
1573 {
1574 modlist_t mod;
1575
1576 TAILQ_FOREACH(mod, &found_modules, link) {
1577 if (strcmp(mod->name, name) == 0 &&
1578 (ver == 0 || mod->version == ver))
1579 return (mod);
1580 }
1581 return (NULL);
1582 }
1583
1584 static modlist_t
modlist_lookup2(const char * name,const struct mod_depend * verinfo)1585 modlist_lookup2(const char *name, const struct mod_depend *verinfo)
1586 {
1587 modlist_t mod, bestmod;
1588 int ver;
1589
1590 if (verinfo == NULL)
1591 return (modlist_lookup(name, 0));
1592 bestmod = NULL;
1593 TAILQ_FOREACH(mod, &found_modules, link) {
1594 if (strcmp(mod->name, name) != 0)
1595 continue;
1596 ver = mod->version;
1597 if (ver == verinfo->md_ver_preferred)
1598 return (mod);
1599 if (ver >= verinfo->md_ver_minimum &&
1600 ver <= verinfo->md_ver_maximum &&
1601 (bestmod == NULL || ver > bestmod->version))
1602 bestmod = mod;
1603 }
1604 return (bestmod);
1605 }
1606
1607 static modlist_t
modlist_newmodule(const char * modname,int version,linker_file_t container)1608 modlist_newmodule(const char *modname, int version, linker_file_t container)
1609 {
1610 modlist_t mod;
1611
1612 mod = malloc(sizeof(struct modlist), M_LINKER, M_NOWAIT | M_ZERO);
1613 if (mod == NULL)
1614 panic("no memory for module list");
1615 mod->container = container;
1616 mod->name = modname;
1617 mod->version = version;
1618 TAILQ_INSERT_TAIL(&found_modules, mod, link);
1619 return (mod);
1620 }
1621
1622 static void
linker_addmodules(linker_file_t lf,struct mod_metadata ** start,struct mod_metadata ** stop,int preload)1623 linker_addmodules(linker_file_t lf, struct mod_metadata **start,
1624 struct mod_metadata **stop, int preload)
1625 {
1626 struct mod_metadata *mp, **mdp;
1627 const char *modname;
1628 int ver;
1629
1630 for (mdp = start; mdp < stop; mdp++) {
1631 mp = *mdp;
1632 if (mp->md_type != MDT_VERSION)
1633 continue;
1634 modname = mp->md_cval;
1635 ver = ((const struct mod_version *)mp->md_data)->mv_version;
1636 if (modlist_lookup(modname, ver) != NULL) {
1637 printf("module %s already present!\n", modname);
1638 /* XXX what can we do? this is a build error. :-( */
1639 continue;
1640 }
1641 modlist_newmodule(modname, ver, lf);
1642 }
1643 }
1644
1645 static void
linker_preload(void * arg)1646 linker_preload(void *arg)
1647 {
1648 caddr_t modptr;
1649 const char *modname, *nmodname;
1650 char *modtype;
1651 linker_file_t lf, nlf;
1652 linker_class_t lc;
1653 int error;
1654 linker_file_list_t loaded_files;
1655 linker_file_list_t depended_files;
1656 struct mod_metadata *mp, *nmp;
1657 struct mod_metadata **start, **stop, **mdp, **nmdp;
1658 const struct mod_depend *verinfo;
1659 int nver;
1660 int resolves;
1661 modlist_t mod;
1662 struct sysinit **si_start, **si_stop;
1663
1664 TAILQ_INIT(&loaded_files);
1665 TAILQ_INIT(&depended_files);
1666 TAILQ_INIT(&found_modules);
1667 error = 0;
1668
1669 modptr = NULL;
1670 sx_xlock(&kld_sx);
1671 while ((modptr = preload_search_next_name(modptr)) != NULL) {
1672 modname = (char *)preload_search_info(modptr, MODINFO_NAME);
1673 modtype = (char *)preload_search_info(modptr, MODINFO_TYPE);
1674 if (modname == NULL) {
1675 printf("Preloaded module at %p does not have a"
1676 " name!\n", modptr);
1677 continue;
1678 }
1679 if (modtype == NULL) {
1680 printf("Preloaded module at %p does not have a type!\n",
1681 modptr);
1682 continue;
1683 }
1684 if (bootverbose)
1685 printf("Preloaded %s \"%s\" at %p.\n", modtype, modname,
1686 modptr);
1687 lf = NULL;
1688 TAILQ_FOREACH(lc, &classes, link) {
1689 error = LINKER_LINK_PRELOAD(lc, modname, &lf);
1690 if (!error)
1691 break;
1692 lf = NULL;
1693 }
1694 if (lf)
1695 TAILQ_INSERT_TAIL(&loaded_files, lf, loaded);
1696 }
1697
1698 /*
1699 * First get a list of stuff in the kernel.
1700 */
1701 if (linker_file_lookup_set(linker_kernel_file, MDT_SETNAME, &start,
1702 &stop, NULL) == 0)
1703 linker_addmodules(linker_kernel_file, start, stop, 1);
1704
1705 /*
1706 * This is a once-off kinky bubble sort to resolve relocation
1707 * dependency requirements.
1708 */
1709 restart:
1710 TAILQ_FOREACH(lf, &loaded_files, loaded) {
1711 error = linker_file_lookup_set(lf, MDT_SETNAME, &start,
1712 &stop, NULL);
1713 /*
1714 * First, look to see if we would successfully link with this
1715 * stuff.
1716 */
1717 resolves = 1; /* unless we know otherwise */
1718 if (!error) {
1719 for (mdp = start; mdp < stop; mdp++) {
1720 mp = *mdp;
1721 if (mp->md_type != MDT_DEPEND)
1722 continue;
1723 modname = mp->md_cval;
1724 verinfo = mp->md_data;
1725 for (nmdp = start; nmdp < stop; nmdp++) {
1726 nmp = *nmdp;
1727 if (nmp->md_type != MDT_VERSION)
1728 continue;
1729 nmodname = nmp->md_cval;
1730 if (strcmp(modname, nmodname) == 0)
1731 break;
1732 }
1733 if (nmdp < stop) /* it's a self reference */
1734 continue;
1735
1736 /*
1737 * ok, the module isn't here yet, we
1738 * are not finished
1739 */
1740 if (modlist_lookup2(modname, verinfo) == NULL)
1741 resolves = 0;
1742 }
1743 }
1744 /*
1745 * OK, if we found our modules, we can link. So, "provide"
1746 * the modules inside and add it to the end of the link order
1747 * list.
1748 */
1749 if (resolves) {
1750 if (!error) {
1751 for (mdp = start; mdp < stop; mdp++) {
1752 mp = *mdp;
1753 if (mp->md_type != MDT_VERSION)
1754 continue;
1755 modname = mp->md_cval;
1756 nver = ((const struct mod_version *)
1757 mp->md_data)->mv_version;
1758 if (modlist_lookup(modname,
1759 nver) != NULL) {
1760 printf("module %s already"
1761 " present!\n", modname);
1762 TAILQ_REMOVE(&loaded_files,
1763 lf, loaded);
1764 linker_file_unload(lf,
1765 LINKER_UNLOAD_FORCE);
1766 /* we changed tailq next ptr */
1767 goto restart;
1768 }
1769 modlist_newmodule(modname, nver, lf);
1770 }
1771 }
1772 TAILQ_REMOVE(&loaded_files, lf, loaded);
1773 TAILQ_INSERT_TAIL(&depended_files, lf, loaded);
1774 /*
1775 * Since we provided modules, we need to restart the
1776 * sort so that the previous files that depend on us
1777 * have a chance. Also, we've busted the tailq next
1778 * pointer with the REMOVE.
1779 */
1780 goto restart;
1781 }
1782 }
1783
1784 /*
1785 * At this point, we check to see what could not be resolved..
1786 */
1787 while ((lf = TAILQ_FIRST(&loaded_files)) != NULL) {
1788 TAILQ_REMOVE(&loaded_files, lf, loaded);
1789 printf("KLD file %s is missing dependencies\n", lf->filename);
1790 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
1791 }
1792
1793 /*
1794 * We made it. Finish off the linking in the order we determined.
1795 */
1796 TAILQ_FOREACH_SAFE(lf, &depended_files, loaded, nlf) {
1797 if (linker_kernel_file) {
1798 linker_kernel_file->refs++;
1799 linker_file_add_dependency(lf, linker_kernel_file);
1800 }
1801 error = linker_file_lookup_set(lf, MDT_SETNAME, &start,
1802 &stop, NULL);
1803 if (!error) {
1804 for (mdp = start; mdp < stop; mdp++) {
1805 mp = *mdp;
1806 if (mp->md_type != MDT_DEPEND)
1807 continue;
1808 modname = mp->md_cval;
1809 verinfo = mp->md_data;
1810 mod = modlist_lookup2(modname, verinfo);
1811 if (mod == NULL) {
1812 printf("KLD file %s - cannot find "
1813 "dependency \"%s\"\n",
1814 lf->filename, modname);
1815 goto fail;
1816 }
1817 /* Don't count self-dependencies */
1818 if (lf == mod->container)
1819 continue;
1820 mod->container->refs++;
1821 linker_file_add_dependency(lf, mod->container);
1822 }
1823 }
1824 /*
1825 * Now do relocation etc using the symbol search paths
1826 * established by the dependencies
1827 */
1828 error = LINKER_LINK_PRELOAD_FINISH(lf);
1829 if (error) {
1830 printf("KLD file %s - could not finalize loading\n",
1831 lf->filename);
1832 goto fail;
1833 }
1834 linker_file_register_modules(lf);
1835 if (!TAILQ_EMPTY(&lf->modules))
1836 lf->flags |= LINKER_FILE_MODULES;
1837 if (linker_file_lookup_set(lf, "sysinit_set", &si_start,
1838 &si_stop, NULL) == 0)
1839 sysinit_add(si_start, si_stop);
1840 linker_file_register_sysctls(lf, true);
1841 lf->flags |= LINKER_FILE_LINKED;
1842 continue;
1843 fail:
1844 TAILQ_REMOVE(&depended_files, lf, loaded);
1845 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
1846 }
1847 sx_xunlock(&kld_sx);
1848 /* woohoo! we made it! */
1849 }
1850 SYSINIT(preload, SI_SUB_KLD, SI_ORDER_MIDDLE, linker_preload, NULL);
1851
1852 static void
linker_mountroot(void * arg __unused)1853 linker_mountroot(void *arg __unused)
1854 {
1855 linker_file_t lf;
1856
1857 sx_xlock(&kld_sx);
1858 TAILQ_FOREACH (lf, &linker_files, link) {
1859 linker_ctf_load_file(lf);
1860 }
1861 sx_xunlock(&kld_sx);
1862 }
1863 EVENTHANDLER_DEFINE(mountroot, linker_mountroot, NULL, 0);
1864
1865 /*
1866 * Handle preload files that failed to load any modules.
1867 */
1868 static void
linker_preload_finish(void * arg)1869 linker_preload_finish(void *arg)
1870 {
1871 linker_file_t lf, nlf;
1872
1873 sx_xlock(&kld_sx);
1874 TAILQ_FOREACH_SAFE(lf, &linker_files, link, nlf) {
1875 if (lf == linker_kernel_file)
1876 continue;
1877
1878 /*
1879 * If all of the modules in this file failed to load, unload
1880 * the file and return an error of ENOEXEC. (Parity with
1881 * linker_load_file.)
1882 */
1883 if ((lf->flags & LINKER_FILE_MODULES) != 0 &&
1884 TAILQ_EMPTY(&lf->modules)) {
1885 linker_file_unload(lf, LINKER_UNLOAD_FORCE);
1886 continue;
1887 }
1888
1889 lf->flags &= ~LINKER_FILE_MODULES;
1890 lf->userrefs++; /* so we can (try to) kldunload it */
1891 }
1892 sx_xunlock(&kld_sx);
1893 }
1894
1895 /*
1896 * Attempt to run after all DECLARE_MODULE SYSINITs. Unfortunately they can be
1897 * scheduled at any subsystem and order, so run this as late as possible. init
1898 * becomes runnable in SI_SUB_KTHREAD_INIT, so go slightly before that.
1899 */
1900 SYSINIT(preload_finish, SI_SUB_KTHREAD_INIT - 100, SI_ORDER_MIDDLE,
1901 linker_preload_finish, NULL);
1902
1903 /*
1904 * Search for a not-loaded module by name.
1905 *
1906 * Modules may be found in the following locations:
1907 *
1908 * - preloaded (result is just the module name) - on disk (result is full path
1909 * to module)
1910 *
1911 * If the module name is qualified in any way (contains path, etc.) the we
1912 * simply return a copy of it.
1913 *
1914 * The search path can be manipulated via sysctl. Note that we use the ';'
1915 * character as a separator to be consistent with the bootloader.
1916 */
1917
1918 static char linker_hintfile[] = "linker.hints";
1919 static char linker_path[MAXPATHLEN] = "/boot/kernel;/boot/modules";
1920
1921 SYSCTL_STRING(_kern, OID_AUTO, module_path, CTLFLAG_RWTUN, linker_path,
1922 sizeof(linker_path), "module load search path");
1923
1924 TUNABLE_STR("module_path", linker_path, sizeof(linker_path));
1925
1926 static const char * const linker_ext_list[] = {
1927 "",
1928 ".ko",
1929 NULL
1930 };
1931
1932 /*
1933 * Check if file actually exists either with or without extension listed in
1934 * the linker_ext_list. (probably should be generic for the rest of the
1935 * kernel)
1936 */
1937 static char *
linker_lookup_file(const char * path,int pathlen,const char * name,int namelen,struct vattr * vap)1938 linker_lookup_file(const char *path, int pathlen, const char *name,
1939 int namelen, struct vattr *vap)
1940 {
1941 struct nameidata nd;
1942 struct thread *td = curthread; /* XXX */
1943 const char * const *cpp, *sep;
1944 char *result;
1945 int error, len, extlen, reclen, flags;
1946 __enum_uint8(vtype) type;
1947
1948 extlen = 0;
1949 for (cpp = linker_ext_list; *cpp; cpp++) {
1950 len = strlen(*cpp);
1951 if (len > extlen)
1952 extlen = len;
1953 }
1954 extlen++; /* trailing '\0' */
1955 sep = (path[pathlen - 1] != '/') ? "/" : "";
1956
1957 reclen = pathlen + strlen(sep) + namelen + extlen + 1;
1958 result = malloc(reclen, M_LINKER, M_WAITOK);
1959 for (cpp = linker_ext_list; *cpp; cpp++) {
1960 snprintf(result, reclen, "%.*s%s%.*s%s", pathlen, path, sep,
1961 namelen, name, *cpp);
1962 /*
1963 * Attempt to open the file, and return the path if
1964 * we succeed and it's a regular file.
1965 */
1966 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, result);
1967 flags = FREAD;
1968 error = vn_open(&nd, &flags, 0, NULL);
1969 if (error == 0) {
1970 NDFREE_PNBUF(&nd);
1971 type = nd.ni_vp->v_type;
1972 if (vap)
1973 VOP_GETATTR(nd.ni_vp, vap, td->td_ucred);
1974 VOP_UNLOCK(nd.ni_vp);
1975 vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
1976 if (type == VREG)
1977 return (result);
1978 }
1979 }
1980 free(result, M_LINKER);
1981 return (NULL);
1982 }
1983
1984 #define INT_ALIGN(base, ptr) ptr = \
1985 (base) + roundup2((ptr) - (base), sizeof(int))
1986
1987 /*
1988 * Lookup KLD which contains requested module in the "linker.hints" file. If
1989 * version specification is available, then try to find the best KLD.
1990 * Otherwise just find the latest one.
1991 */
1992 static char *
linker_hints_lookup(const char * path,int pathlen,const char * modname,int modnamelen,const struct mod_depend * verinfo)1993 linker_hints_lookup(const char *path, int pathlen, const char *modname,
1994 int modnamelen, const struct mod_depend *verinfo)
1995 {
1996 struct thread *td = curthread; /* XXX */
1997 struct ucred *cred = td ? td->td_ucred : NULL;
1998 struct nameidata nd;
1999 struct vattr vattr, mattr;
2000 const char *best, *sep;
2001 u_char *hints = NULL;
2002 u_char *cp, *recptr, *bufend, *result, *pathbuf;
2003 int error, ival, bestver, *intp, found, flags, clen, blen;
2004 ssize_t reclen;
2005
2006 result = NULL;
2007 bestver = found = 0;
2008
2009 sep = (path[pathlen - 1] != '/') ? "/" : "";
2010 reclen = imax(modnamelen, strlen(linker_hintfile)) + pathlen +
2011 strlen(sep) + 1;
2012 pathbuf = malloc(reclen, M_LINKER, M_WAITOK);
2013 snprintf(pathbuf, reclen, "%.*s%s%s", pathlen, path, sep,
2014 linker_hintfile);
2015
2016 NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, pathbuf);
2017 flags = FREAD;
2018 error = vn_open(&nd, &flags, 0, NULL);
2019 if (error)
2020 goto bad;
2021 NDFREE_PNBUF(&nd);
2022 if (nd.ni_vp->v_type != VREG)
2023 goto bad;
2024 best = cp = NULL;
2025 error = VOP_GETATTR(nd.ni_vp, &vattr, cred);
2026 if (error)
2027 goto bad;
2028 /*
2029 * XXX: we need to limit this number to some reasonable value
2030 */
2031 if (vattr.va_size > LINKER_HINTS_MAX) {
2032 printf("linker.hints file too large %ld\n", (long)vattr.va_size);
2033 goto bad;
2034 }
2035 if (vattr.va_size < sizeof(ival)) {
2036 printf("linker.hints file truncated\n");
2037 goto bad;
2038 }
2039 hints = malloc(vattr.va_size, M_TEMP, M_WAITOK);
2040 error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)hints, vattr.va_size, 0,
2041 UIO_SYSSPACE, IO_NODELOCKED, cred, NOCRED, &reclen, td);
2042 if (error)
2043 goto bad;
2044 VOP_UNLOCK(nd.ni_vp);
2045 vn_close(nd.ni_vp, FREAD, cred, td);
2046 nd.ni_vp = NULL;
2047 if (reclen != 0) {
2048 printf("can't read %zd\n", reclen);
2049 goto bad;
2050 }
2051 intp = (int *)hints;
2052 ival = *intp++;
2053 if (ival != LINKER_HINTS_VERSION) {
2054 printf("linker.hints file version mismatch %d\n", ival);
2055 goto bad;
2056 }
2057 bufend = hints + vattr.va_size;
2058 recptr = (u_char *)intp;
2059 clen = blen = 0;
2060 while (recptr < bufend && !found) {
2061 intp = (int *)recptr;
2062 reclen = *intp++;
2063 ival = *intp++;
2064 cp = (char *)intp;
2065 switch (ival) {
2066 case MDT_VERSION:
2067 clen = *cp++;
2068 if (clen != modnamelen || bcmp(cp, modname, clen) != 0)
2069 break;
2070 cp += clen;
2071 INT_ALIGN(hints, cp);
2072 ival = *(int *)cp;
2073 cp += sizeof(int);
2074 clen = *cp++;
2075 if (verinfo == NULL ||
2076 ival == verinfo->md_ver_preferred) {
2077 found = 1;
2078 break;
2079 }
2080 if (ival >= verinfo->md_ver_minimum &&
2081 ival <= verinfo->md_ver_maximum &&
2082 ival > bestver) {
2083 bestver = ival;
2084 best = cp;
2085 blen = clen;
2086 }
2087 break;
2088 default:
2089 break;
2090 }
2091 recptr += reclen + sizeof(int);
2092 }
2093 /*
2094 * Finally check if KLD is in the place
2095 */
2096 if (found)
2097 result = linker_lookup_file(path, pathlen, cp, clen, &mattr);
2098 else if (best)
2099 result = linker_lookup_file(path, pathlen, best, blen, &mattr);
2100
2101 /*
2102 * KLD is newer than hints file. What we should do now?
2103 */
2104 if (result && timespeccmp(&mattr.va_mtime, &vattr.va_mtime, >))
2105 printf("warning: KLD '%s' is newer than the linker.hints"
2106 " file\n", result);
2107 bad:
2108 free(pathbuf, M_LINKER);
2109 if (hints)
2110 free(hints, M_TEMP);
2111 if (nd.ni_vp != NULL) {
2112 VOP_UNLOCK(nd.ni_vp);
2113 vn_close(nd.ni_vp, FREAD, cred, td);
2114 }
2115 /*
2116 * If nothing found or hints is absent - fallback to the old
2117 * way by using "kldname[.ko]" as module name.
2118 */
2119 if (!found && !bestver && result == NULL)
2120 result = linker_lookup_file(path, pathlen, modname,
2121 modnamelen, NULL);
2122 return (result);
2123 }
2124
2125 /*
2126 * Lookup KLD which contains requested module in the all directories.
2127 */
2128 static char *
linker_search_module(const char * modname,int modnamelen,const struct mod_depend * verinfo)2129 linker_search_module(const char *modname, int modnamelen,
2130 const struct mod_depend *verinfo)
2131 {
2132 char *cp, *ep, *result;
2133
2134 /*
2135 * traverse the linker path
2136 */
2137 for (cp = linker_path; *cp; cp = ep + 1) {
2138 /* find the end of this component */
2139 for (ep = cp; (*ep != 0) && (*ep != ';'); ep++);
2140 result = linker_hints_lookup(cp, ep - cp, modname,
2141 modnamelen, verinfo);
2142 if (result != NULL)
2143 return (result);
2144 if (*ep == 0)
2145 break;
2146 }
2147 return (NULL);
2148 }
2149
2150 /*
2151 * Search for module in all directories listed in the linker_path.
2152 */
2153 static char *
linker_search_kld(const char * name)2154 linker_search_kld(const char *name)
2155 {
2156 char *cp, *ep, *result;
2157 int len;
2158
2159 /* qualified at all? */
2160 if (strchr(name, '/'))
2161 return (strdup(name, M_LINKER));
2162
2163 /* traverse the linker path */
2164 len = strlen(name);
2165 for (ep = linker_path; *ep; ep++) {
2166 cp = ep;
2167 /* find the end of this component */
2168 for (; *ep != 0 && *ep != ';'; ep++);
2169 result = linker_lookup_file(cp, ep - cp, name, len, NULL);
2170 if (result != NULL)
2171 return (result);
2172 }
2173 return (NULL);
2174 }
2175
2176 static const char *
linker_basename(const char * path)2177 linker_basename(const char *path)
2178 {
2179 const char *filename;
2180
2181 filename = strrchr(path, '/');
2182 if (filename == NULL)
2183 return path;
2184 if (filename[1])
2185 filename++;
2186 return (filename);
2187 }
2188
2189 #if defined(HWPMC_HOOKS) || defined(HWT_HOOKS)
2190 /*
2191 * Inform hwpmc about the set of kernel modules currently loaded.
2192 */
2193 void *
linker_hwpmc_list_objects(void)2194 linker_hwpmc_list_objects(void)
2195 {
2196 linker_file_t lf;
2197 struct pmckern_map_in *kobase;
2198 int i, nmappings;
2199
2200 nmappings = 0;
2201 sx_slock(&kld_sx);
2202 TAILQ_FOREACH(lf, &linker_files, link)
2203 nmappings++;
2204
2205 /* Allocate nmappings + 1 entries. */
2206 kobase = malloc((nmappings + 1) * sizeof(struct pmckern_map_in),
2207 M_LINKER, M_WAITOK | M_ZERO);
2208 i = 0;
2209 TAILQ_FOREACH(lf, &linker_files, link) {
2210 /* Save the info for this linker file. */
2211 kobase[i].pm_file = lf->pathname;
2212 kobase[i].pm_address = (uintptr_t)lf->address;
2213 i++;
2214 }
2215 sx_sunlock(&kld_sx);
2216
2217 KASSERT(i > 0, ("linker_hpwmc_list_objects: no kernel objects?"));
2218
2219 /* The last entry of the malloced area comprises of all zeros. */
2220 KASSERT(kobase[i].pm_file == NULL,
2221 ("linker_hwpmc_list_objects: last object not NULL"));
2222
2223 return ((void *)kobase);
2224 }
2225 #endif
2226
2227 /* check if root file system is not mounted */
2228 static bool
linker_root_mounted(void)2229 linker_root_mounted(void)
2230 {
2231 struct pwd *pwd;
2232 bool ret;
2233
2234 if (rootvnode == NULL)
2235 return (false);
2236
2237 pwd = pwd_hold(curthread);
2238 ret = pwd->pwd_rdir != NULL;
2239 pwd_drop(pwd);
2240 return (ret);
2241 }
2242
2243 /*
2244 * Find a file which contains given module and load it, if "parent" is not
2245 * NULL, register a reference to it.
2246 */
2247 static int
linker_load_module(const char * kldname,const char * modname,struct linker_file * parent,const struct mod_depend * verinfo,struct linker_file ** lfpp)2248 linker_load_module(const char *kldname, const char *modname,
2249 struct linker_file *parent, const struct mod_depend *verinfo,
2250 struct linker_file **lfpp)
2251 {
2252 linker_file_t lfdep;
2253 const char *filename;
2254 char *pathname;
2255 int error;
2256
2257 sx_assert(&kld_sx, SA_XLOCKED);
2258 if (modname == NULL) {
2259 /*
2260 * We have to load KLD
2261 */
2262 KASSERT(verinfo == NULL, ("linker_load_module: verinfo"
2263 " is not NULL"));
2264 if (!linker_root_mounted())
2265 return (ENXIO);
2266 pathname = linker_search_kld(kldname);
2267 } else {
2268 if (modlist_lookup2(modname, verinfo) != NULL)
2269 return (EEXIST);
2270 if (!linker_root_mounted())
2271 return (ENXIO);
2272 if (kldname != NULL)
2273 pathname = strdup(kldname, M_LINKER);
2274 else
2275 /*
2276 * Need to find a KLD with required module
2277 */
2278 pathname = linker_search_module(modname,
2279 strlen(modname), verinfo);
2280 }
2281 if (pathname == NULL)
2282 return (ENOENT);
2283
2284 /*
2285 * Can't load more than one file with the same basename XXX:
2286 * Actually it should be possible to have multiple KLDs with
2287 * the same basename but different path because they can
2288 * provide different versions of the same modules.
2289 */
2290 filename = linker_basename(pathname);
2291 if (linker_find_file_by_name(filename))
2292 error = EEXIST;
2293 else do {
2294 error = linker_load_file(pathname, &lfdep);
2295 if (error)
2296 break;
2297 if (modname && verinfo &&
2298 modlist_lookup2(modname, verinfo) == NULL) {
2299 linker_file_unload(lfdep, LINKER_UNLOAD_FORCE);
2300 error = ENOENT;
2301 break;
2302 }
2303 if (parent)
2304 linker_file_add_dependency(parent, lfdep);
2305 if (lfpp)
2306 *lfpp = lfdep;
2307 } while (0);
2308 free(pathname, M_LINKER);
2309 return (error);
2310 }
2311
2312 /*
2313 * This routine is responsible for finding dependencies of userland initiated
2314 * kldload(2)'s of files.
2315 */
2316 int
linker_load_dependencies(linker_file_t lf)2317 linker_load_dependencies(linker_file_t lf)
2318 {
2319 linker_file_t lfdep;
2320 struct mod_metadata **start, **stop, **mdp, **nmdp;
2321 struct mod_metadata *mp, *nmp;
2322 const struct mod_depend *verinfo;
2323 modlist_t mod;
2324 const char *modname, *nmodname;
2325 int ver, error = 0;
2326
2327 /*
2328 * All files are dependent on /kernel.
2329 */
2330 sx_assert(&kld_sx, SA_XLOCKED);
2331 if (linker_kernel_file) {
2332 linker_kernel_file->refs++;
2333 linker_file_add_dependency(lf, linker_kernel_file);
2334 }
2335 if (linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop,
2336 NULL) != 0)
2337 return (0);
2338 for (mdp = start; mdp < stop; mdp++) {
2339 mp = *mdp;
2340 if (mp->md_type != MDT_VERSION)
2341 continue;
2342 modname = mp->md_cval;
2343 ver = ((const struct mod_version *)mp->md_data)->mv_version;
2344 mod = modlist_lookup(modname, ver);
2345 if (mod != NULL) {
2346 printf("interface %s.%d already present in the KLD"
2347 " '%s'!\n", modname, ver,
2348 mod->container->filename);
2349 return (EEXIST);
2350 }
2351 }
2352
2353 for (mdp = start; mdp < stop; mdp++) {
2354 mp = *mdp;
2355 if (mp->md_type != MDT_DEPEND)
2356 continue;
2357 modname = mp->md_cval;
2358 verinfo = mp->md_data;
2359 nmodname = NULL;
2360 for (nmdp = start; nmdp < stop; nmdp++) {
2361 nmp = *nmdp;
2362 if (nmp->md_type != MDT_VERSION)
2363 continue;
2364 nmodname = nmp->md_cval;
2365 if (strcmp(modname, nmodname) == 0)
2366 break;
2367 }
2368 if (nmdp < stop)/* early exit, it's a self reference */
2369 continue;
2370 mod = modlist_lookup2(modname, verinfo);
2371 if (mod) { /* woohoo, it's loaded already */
2372 lfdep = mod->container;
2373 lfdep->refs++;
2374 linker_file_add_dependency(lf, lfdep);
2375 continue;
2376 }
2377 error = linker_load_module(NULL, modname, lf, verinfo, NULL);
2378 if (error) {
2379 printf("KLD %s: depends on %s - not available or"
2380 " version mismatch\n", lf->filename, modname);
2381 break;
2382 }
2383 }
2384
2385 if (error)
2386 return (error);
2387 linker_addmodules(lf, start, stop, 0);
2388 return (error);
2389 }
2390
2391 static int
sysctl_kern_function_list_iterate(const char * name,void * opaque)2392 sysctl_kern_function_list_iterate(const char *name, void *opaque)
2393 {
2394 struct sysctl_req *req;
2395
2396 req = opaque;
2397 return (SYSCTL_OUT(req, name, strlen(name) + 1));
2398 }
2399
2400 /*
2401 * Export a nul-separated, double-nul-terminated list of all function names
2402 * in the kernel.
2403 */
2404 static int
sysctl_kern_function_list(SYSCTL_HANDLER_ARGS)2405 sysctl_kern_function_list(SYSCTL_HANDLER_ARGS)
2406 {
2407 linker_file_t lf;
2408 int error;
2409
2410 #ifdef MAC
2411 error = mac_kld_check_stat(req->td->td_ucred);
2412 if (error)
2413 return (error);
2414 #endif
2415 error = sysctl_wire_old_buffer(req, 0);
2416 if (error != 0)
2417 return (error);
2418 sx_xlock(&kld_sx);
2419 TAILQ_FOREACH(lf, &linker_files, link) {
2420 error = LINKER_EACH_FUNCTION_NAME(lf,
2421 sysctl_kern_function_list_iterate, req);
2422 if (error) {
2423 sx_xunlock(&kld_sx);
2424 return (error);
2425 }
2426 }
2427 sx_xunlock(&kld_sx);
2428 return (SYSCTL_OUT(req, "", 1));
2429 }
2430
2431 SYSCTL_PROC(_kern, OID_AUTO, function_list,
2432 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
2433 sysctl_kern_function_list, "",
2434 "kernel function list");
2435