1 /*
2 * SPDX-License-Identifier: CDDL 1.0
3 *
4 * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org>
5 * Copyright (c) 2023 The FreeBSD Foundation
6 *
7 * Portions of this software were developed by Christos Margiolis
8 * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
9 */
10
11 #include <sys/param.h>
12 #include <sys/systm.h>
13 #include <sys/conf.h>
14 #include <sys/kernel.h>
15 #include <sys/linker.h>
16 #include <sys/module.h>
17
18 #include <sys/dtrace.h>
19
20 #include "kinst.h"
21
22 MALLOC_DEFINE(M_KINST, "kinst", "Kernel Instruction Tracing");
23
24 static d_open_t kinst_open;
25 static d_close_t kinst_close;
26 static d_ioctl_t kinst_ioctl;
27
28 static void kinst_provide_module(void *, modctl_t *);
29 static void kinst_getargdesc(void *, dtrace_id_t, void *,
30 dtrace_argdesc_t *);
31 static void kinst_destroy(void *, dtrace_id_t, void *);
32 static void kinst_enable(void *, dtrace_id_t, void *);
33 static void kinst_disable(void *, dtrace_id_t, void *);
34 static int kinst_load(void *);
35 static int kinst_unload(void *);
36 static int kinst_modevent(module_t, int, void *);
37
38 static dtrace_pattr_t kinst_attr = {
39 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
40 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
41 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
42 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
43 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
44 };
45
46 static const dtrace_pops_t kinst_pops = {
47 .dtps_provide = NULL,
48 .dtps_provide_module = kinst_provide_module,
49 .dtps_enable = kinst_enable,
50 .dtps_disable = kinst_disable,
51 .dtps_suspend = NULL,
52 .dtps_resume = NULL,
53 .dtps_getargdesc = kinst_getargdesc,
54 .dtps_getargval = NULL,
55 .dtps_usermode = NULL,
56 .dtps_destroy = kinst_destroy
57 };
58
59 static struct cdevsw kinst_cdevsw = {
60 .d_name = "kinst",
61 .d_version = D_VERSION,
62 .d_flags = D_TRACKCLOSE,
63 .d_open = kinst_open,
64 .d_close = kinst_close,
65 .d_ioctl = kinst_ioctl,
66 };
67
68 static dtrace_provider_id_t kinst_id;
69 struct kinst_probe_list *kinst_probetab;
70 static struct cdev *kinst_cdev;
71
72 /*
73 * Tracing memcpy() will crash the kernel when kinst tries to trace an instance
74 * of the memcpy() calls in kinst_invop(). To fix this, we can use
75 * kinst_memcpy() in those cases, with its arguments marked as 'volatile' to
76 * "outsmart" the compiler and avoid having it replaced by a regular memcpy().
77 */
78 volatile void *
kinst_memcpy(volatile void * dst,volatile const void * src,size_t len)79 kinst_memcpy(volatile void *dst, volatile const void *src, size_t len)
80 {
81 volatile const unsigned char *src0;
82 volatile unsigned char *dst0;
83
84 src0 = src;
85 dst0 = dst;
86
87 while (len--)
88 *dst0++ = *src0++;
89
90 return (dst);
91 }
92
93 bool
kinst_excluded(const char * name)94 kinst_excluded(const char *name)
95 {
96 if (kinst_md_excluded(name))
97 return (true);
98
99 /*
100 * cpu_switch() can cause a crash if it modifies the value of curthread
101 * while in probe context.
102 */
103 if (strcmp(name, "cpu_switch") == 0)
104 return (true);
105
106 /*
107 * Anything beginning with "dtrace_" may be called from probe context
108 * unless it explicitly indicates that it won't be called from probe
109 * context by using the prefix "dtrace_safe_".
110 */
111 if (strncmp(name, "dtrace_", strlen("dtrace_")) == 0 &&
112 strncmp(name, "dtrace_safe_", strlen("dtrace_safe_")) != 0)
113 return (true);
114
115 /*
116 * Omit instrumentation of functions that are probably in DDB. It
117 * makes it too hard to debug broken kinst.
118 *
119 * NB: kdb_enter() can be excluded, but its call to printf() can't be.
120 * This is generally OK since we're not yet in debugging context.
121 */
122 if (strncmp(name, "db_", strlen("db_")) == 0 ||
123 strncmp(name, "kdb_", strlen("kdb_")) == 0)
124 return (true);
125
126 /*
127 * Lock owner methods may be called from probe context.
128 */
129 if (strcmp(name, "owner_mtx") == 0 ||
130 strcmp(name, "owner_rm") == 0 ||
131 strcmp(name, "owner_rw") == 0 ||
132 strcmp(name, "owner_sx") == 0)
133 return (true);
134
135 /*
136 * The KMSAN runtime can't be instrumented safely.
137 */
138 if (strncmp(name, "__msan", 6) == 0 ||
139 strncmp(name, "kmsan_", 6) == 0)
140 return (1);
141
142 /*
143 * When DTrace is built into the kernel we need to exclude the kinst
144 * functions from instrumentation.
145 */
146 #ifndef _KLD_MODULE
147 if (strncmp(name, "kinst_", strlen("kinst_")) == 0)
148 return (true);
149 #endif
150
151 if (strcmp(name, "trap_check") == 0)
152 return (true);
153
154 return (false);
155 }
156
157 void
kinst_probe_create(struct kinst_probe * kp,linker_file_t lf)158 kinst_probe_create(struct kinst_probe *kp, linker_file_t lf)
159 {
160 kp->kp_id = dtrace_probe_create(kinst_id, lf->filename,
161 kp->kp_func, kp->kp_name, 3, kp);
162
163 LIST_INSERT_HEAD(KINST_GETPROBE(kp->kp_patchpoint), kp, kp_hashnext);
164 }
165
166 static int
kinst_open(struct cdev * dev __unused,int oflags __unused,int devtype __unused,struct thread * td __unused)167 kinst_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused,
168 struct thread *td __unused)
169 {
170 return (0);
171 }
172
173 static int
kinst_close(struct cdev * dev __unused,int fflag __unused,int devtype __unused,struct thread * td __unused)174 kinst_close(struct cdev *dev __unused, int fflag __unused, int devtype __unused,
175 struct thread *td __unused)
176 {
177 dtrace_condense(kinst_id);
178 return (0);
179 }
180
181 static int
kinst_linker_file_cb(linker_file_t lf,void * arg)182 kinst_linker_file_cb(linker_file_t lf, void *arg)
183 {
184 dtrace_kinst_probedesc_t *pd;
185
186 pd = arg;
187 if (pd->kpd_mod[0] != '\0' && strcmp(pd->kpd_mod, lf->filename) != 0)
188 return (0);
189
190 /*
191 * Invoke kinst_make_probe_function() once for each function symbol in
192 * the module "lf".
193 */
194 return (linker_file_function_listall(lf, kinst_make_probe, arg));
195 }
196
197 static int
kinst_ioctl(struct cdev * dev __unused,u_long cmd,caddr_t addr,int flags __unused,struct thread * td __unused)198 kinst_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t addr,
199 int flags __unused, struct thread *td __unused)
200 {
201 dtrace_kinst_probedesc_t *pd;
202 int error = 0;
203
204 switch (cmd) {
205 case KINSTIOC_MAKEPROBE:
206 pd = (dtrace_kinst_probedesc_t *)addr;
207 pd->kpd_func[sizeof(pd->kpd_func) - 1] = '\0';
208 pd->kpd_mod[sizeof(pd->kpd_mod) - 1] = '\0';
209
210 /* Loop over all functions in the kernel and loaded modules. */
211 error = linker_file_foreach(kinst_linker_file_cb, pd);
212 break;
213 default:
214 error = ENOTTY;
215 break;
216 }
217
218 return (error);
219 }
220
221 static void
kinst_provide_module(void * arg,modctl_t * lf)222 kinst_provide_module(void *arg, modctl_t *lf)
223 {
224 }
225
226 static void
kinst_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)227 kinst_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
228 {
229 desc->dtargd_ndx = DTRACE_ARGNONE;
230 }
231
232 static void
kinst_destroy(void * arg,dtrace_id_t id,void * parg)233 kinst_destroy(void *arg, dtrace_id_t id, void *parg)
234 {
235 struct kinst_probe *kp = parg;
236
237 LIST_REMOVE(kp, kp_hashnext);
238 #ifndef __amd64__
239 kinst_trampoline_dealloc(kp->kp_tramp);
240 #endif
241 free(kp, M_KINST);
242 }
243
244 static void
kinst_enable(void * arg,dtrace_id_t id,void * parg)245 kinst_enable(void *arg, dtrace_id_t id, void *parg)
246 {
247 struct kinst_probe *kp = parg;
248 static bool warned = false;
249
250 if (!warned) {
251 KINST_LOG(
252 "kinst: This provider is experimental, exercise caution");
253 warned = true;
254 }
255
256 kinst_patch_tracepoint(kp, kp->kp_patchval);
257 }
258
259 static void
kinst_disable(void * arg,dtrace_id_t id,void * parg)260 kinst_disable(void *arg, dtrace_id_t id, void *parg)
261 {
262 struct kinst_probe *kp = parg;
263
264 kinst_patch_tracepoint(kp, kp->kp_savedval);
265 }
266
267 static int
kinst_load(void * dummy)268 kinst_load(void *dummy)
269 {
270 int error;
271
272 error = kinst_trampoline_init();
273 if (error != 0)
274 return (error);
275 error = kinst_md_init();
276 if (error != 0) {
277 kinst_trampoline_deinit();
278 return (error);
279 }
280
281 error = dtrace_register("kinst", &kinst_attr, DTRACE_PRIV_USER, NULL,
282 &kinst_pops, NULL, &kinst_id);
283 if (error != 0) {
284 kinst_md_deinit();
285 kinst_trampoline_deinit();
286 return (error);
287 }
288 kinst_probetab = malloc(KINST_PROBETAB_MAX *
289 sizeof(struct kinst_probe_list), M_KINST, M_WAITOK | M_ZERO);
290 for (int i = 0; i < KINST_PROBETAB_MAX; i++)
291 LIST_INIT(&kinst_probetab[i]);
292 kinst_cdev = make_dev(&kinst_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
293 "dtrace/kinst");
294 dtrace_invop_add(kinst_invop);
295 return (0);
296 }
297
298 static int
kinst_unload(void * dummy)299 kinst_unload(void *dummy)
300 {
301 free(kinst_probetab, M_KINST);
302 kinst_md_deinit();
303 kinst_trampoline_deinit();
304 dtrace_invop_remove(kinst_invop);
305 destroy_dev(kinst_cdev);
306
307 return (dtrace_unregister(kinst_id));
308 }
309
310 static int
kinst_modevent(module_t mod __unused,int type,void * data __unused)311 kinst_modevent(module_t mod __unused, int type, void *data __unused)
312 {
313 int error = 0;
314
315 switch (type) {
316 case MOD_LOAD:
317 break;
318 case MOD_UNLOAD:
319 break;
320 case MOD_SHUTDOWN:
321 break;
322 default:
323 error = EOPNOTSUPP;
324 break;
325 }
326
327 return (error);
328 }
329
330 SYSINIT(kinst_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_load, NULL);
331 SYSUNINIT(kinst_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, kinst_unload,
332 NULL);
333
334 DEV_MODULE(kinst, kinst_modevent, NULL);
335 MODULE_VERSION(kinst, 1);
336 MODULE_DEPEND(kinst, dtrace, 1, 1, 1);
337 MODULE_DEPEND(kinst, opensolaris, 1, 1, 1);
338