1 /*
2 * SPDX-License-Identifier: CDDL 1.0
3 *
4 * Copyright (c) 2022 Christos Margiolis <christos@FreeBSD.org>
5 * Copyright (c) 2022 Mark Johnston <markj@FreeBSD.org>
6 * Copyright (c) 2023 The FreeBSD Foundation
7 *
8 * Portions of this software were developed by Christos Margiolis
9 * <christos@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
10 */
11
12 #include <sys/param.h>
13
14 #include <sys/dtrace.h>
15 #include <cddl/dev/dtrace/dtrace_cddl.h>
16
17 #include "kinst.h"
18
19 DPCPU_DEFINE_STATIC(struct kinst_cpu_state, kinst_state);
20
21 static int
kinst_emulate(struct trapframe * frame,const struct kinst_probe * kp)22 kinst_emulate(struct trapframe *frame, const struct kinst_probe *kp)
23 {
24 kinst_patchval_t instr = kp->kp_savedval;
25 uint64_t imm;
26 uint8_t cond, reg, bitpos;
27 bool res;
28
29 if (((instr >> 24) & 0x1f) == 0b10000) {
30 /* adr/adrp */
31 reg = instr & 0x1f;
32 imm = (instr >> 29) & 0x3;
33 imm |= ((instr >> 5) & 0x0007ffff) << 2;
34 if (((instr >> 31) & 0x1) == 0) {
35 /* adr */
36 if (imm & 0x0000000000100000)
37 imm |= 0xfffffffffff00000;
38 frame->tf_x[reg] = frame->tf_elr + imm;
39 } else {
40 /* adrp */
41 imm <<= 12;
42 if (imm & 0x0000000100000000)
43 imm |= 0xffffffff00000000;
44 frame->tf_x[reg] = (frame->tf_elr & ~0xfff) + imm;
45 }
46 frame->tf_elr += INSN_SIZE;
47 } else if (((instr >> 26) & 0x3f) == 0b000101) {
48 /* b */
49 imm = instr & 0x03ffffff;
50 if (imm & 0x0000000002000000)
51 imm |= 0xfffffffffe000000;
52 frame->tf_elr += imm << 2;
53 } else if (((instr >> 24) & 0xff) == 0b01010100) {
54 /* b.cond */
55 imm = (instr >> 5) & 0x0007ffff;
56 if (imm & 0x0000000000040000)
57 imm |= 0xfffffffffffc0000;
58 cond = instr & 0xf;
59 switch ((cond >> 1) & 0x7) {
60 case 0b000: /* eq/ne */
61 res = (frame->tf_spsr & PSR_Z) != 0;
62 break;
63 case 0b001: /* cs/cc */
64 res = (frame->tf_spsr & PSR_C) != 0;
65 break;
66 case 0b010: /* mi/pl */
67 res = (frame->tf_spsr & PSR_N) != 0;
68 break;
69 case 0b011: /* vs/vc */
70 res = (frame->tf_spsr & PSR_V) != 0;
71 break;
72 case 0b100: /* hi/ls */
73 res = ((frame->tf_spsr & PSR_C) != 0) &&
74 ((frame->tf_spsr & PSR_Z) == 0);
75 break;
76 case 0b101: /* ge/lt */
77 res = ((frame->tf_spsr & PSR_N) != 0) ==
78 ((frame->tf_spsr & PSR_V) != 0);
79 break;
80 case 0b110: /* gt/le */
81 res = ((frame->tf_spsr & PSR_Z) == 0) &&
82 (((frame->tf_spsr & PSR_N) != 0) ==
83 ((frame->tf_spsr & PSR_V) != 0));
84 break;
85 case 0b111: /* al */
86 res = 1;
87 break;
88 }
89 if ((cond & 0x1) && cond != 0b1111)
90 res = !res;
91 if (res)
92 frame->tf_elr += imm << 2;
93 else
94 frame->tf_elr += INSN_SIZE;
95 } else if (((instr >> 26) & 0x3f) == 0b100101) {
96 /* bl */
97 imm = instr & 0x03ffffff;
98 if (imm & 0x0000000002000000)
99 imm |= 0xfffffffffe000000;
100 frame->tf_lr = frame->tf_elr + INSN_SIZE;
101 frame->tf_elr += imm << 2;
102 } else if (((instr >> 25) & 0x3f) == 0b011010) {
103 /* cbnz/cbz */
104 cond = (instr >> 24) & 0x1;
105 reg = instr & 0x1f;
106 imm = (instr >> 5) & 0x0007ffff;
107 if (imm & 0x0000000000040000)
108 imm |= 0xfffffffffffc0000;
109 if (cond == 1 && frame->tf_x[reg] != 0)
110 /* cbnz */
111 frame->tf_elr += imm << 2;
112 else if (cond == 0 && frame->tf_x[reg] == 0)
113 /* cbz */
114 frame->tf_elr += imm << 2;
115 else
116 frame->tf_elr += INSN_SIZE;
117 } else if (((instr >> 25) & 0x3f) == 0b011011) {
118 /* tbnz/tbz */
119 cond = (instr >> 24) & 0x1;
120 reg = instr & 0x1f;
121 bitpos = (instr >> 19) & 0x1f;
122 bitpos |= ((instr >> 31) & 0x1) << 5;
123 imm = (instr >> 5) & 0x3fff;
124 if (imm & 0x0000000000002000)
125 imm |= 0xffffffffffffe000;
126 if (cond == 1 && (frame->tf_x[reg] & (1 << bitpos)) != 0)
127 /* tbnz */
128 frame->tf_elr += imm << 2;
129 else if (cond == 0 && (frame->tf_x[reg] & (1 << bitpos)) == 0)
130 /* tbz */
131 frame->tf_elr += imm << 2;
132 else
133 frame->tf_elr += INSN_SIZE;
134 }
135
136 return (0);
137 }
138
139 static int
kinst_jump_next_instr(struct trapframe * frame,const struct kinst_probe * kp)140 kinst_jump_next_instr(struct trapframe *frame, const struct kinst_probe *kp)
141 {
142 frame->tf_elr = (register_t)((const uint8_t *)kp->kp_patchpoint +
143 INSN_SIZE);
144
145 return (0);
146 }
147
148 static void
kinst_trampoline_populate(struct kinst_probe * kp)149 kinst_trampoline_populate(struct kinst_probe *kp)
150 {
151 static uint32_t bpt = KINST_PATCHVAL;
152
153 kinst_memcpy(kp->kp_tramp, &kp->kp_savedval, INSN_SIZE);
154 kinst_memcpy(&kp->kp_tramp[INSN_SIZE], &bpt, INSN_SIZE);
155
156 cpu_icache_sync_range(kp->kp_tramp, KINST_TRAMP_SIZE);
157 }
158
159 /*
160 * There are two ways by which an instruction is traced:
161 *
162 * - By using the trampoline.
163 * - By emulating it in software (see kinst_emulate()).
164 *
165 * The trampoline is used for instructions that can be copied and executed
166 * as-is without additional modification. However, instructions that use
167 * PC-relative addressing have to be emulated, because ARM64 doesn't allow
168 * encoding of large displacements in a single instruction, and since we cannot
169 * clobber a register in order to encode the two-instruction sequence needed to
170 * create large displacements, we cannot use the trampoline at all.
171 * Fortunately, the instructions are simple enough to be emulated in just a few
172 * lines of code.
173 *
174 * The problem discussed above also means that, unlike amd64, we cannot encode
175 * a far-jump back from the trampoline to the next instruction. The mechanism
176 * employed to achieve this functionality, is to use a breakpoint instead of a
177 * jump after the copied instruction. This breakpoint is detected and handled
178 * by kinst_invop(), which performs the jump back to the next instruction
179 * manually (see kinst_jump_next_instr()).
180 */
181 int
kinst_invop(uintptr_t addr,struct trapframe * frame,uintptr_t scratch)182 kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch)
183 {
184 solaris_cpu_t *cpu;
185 struct kinst_cpu_state *ks;
186 const struct kinst_probe *kp;
187
188 ks = DPCPU_PTR(kinst_state);
189
190 /*
191 * Detect if the breakpoint was triggered by the trampoline, and
192 * manually set the PC to the next instruction.
193 */
194 if (ks->state == KINST_PROBE_FIRED &&
195 addr == (uintptr_t)(ks->kp->kp_tramp + INSN_SIZE)) {
196 /*
197 * Restore interrupts if they were enabled prior to the first
198 * breakpoint.
199 */
200 if ((ks->status & PSR_I) == 0)
201 frame->tf_spsr &= ~PSR_I;
202 ks->state = KINST_PROBE_ARMED;
203 return (kinst_jump_next_instr(frame, ks->kp));
204 }
205
206 LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) {
207 if ((uintptr_t)kp->kp_patchpoint == addr)
208 break;
209 }
210 if (kp == NULL)
211 return (0);
212
213 cpu = &solaris_cpu[curcpu];
214 cpu->cpu_dtrace_caller = addr;
215 dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0);
216 cpu->cpu_dtrace_caller = 0;
217
218 if (kp->kp_md.emulate)
219 return (kinst_emulate(frame, kp));
220
221 ks->state = KINST_PROBE_FIRED;
222 ks->kp = kp;
223
224 /*
225 * Cache the current SPSR and clear interrupts for the duration
226 * of the double breakpoint.
227 */
228 ks->status = frame->tf_spsr;
229 frame->tf_spsr |= PSR_I;
230 frame->tf_elr = (register_t)kp->kp_tramp;
231
232 return (0);
233 }
234
235 void
kinst_patch_tracepoint(struct kinst_probe * kp,kinst_patchval_t val)236 kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val)
237 {
238 void *addr;
239
240 if (!arm64_get_writable_addr(kp->kp_patchpoint, &addr))
241 panic("%s: Unable to write new instruction", __func__);
242 *(kinst_patchval_t *)addr = val;
243 cpu_icache_sync_range(kp->kp_patchpoint, INSN_SIZE);
244 }
245
246 static void
kinst_instr_dissect(struct kinst_probe * kp)247 kinst_instr_dissect(struct kinst_probe *kp)
248 {
249 struct kinst_probe_md *kpmd;
250 kinst_patchval_t instr = kp->kp_savedval;
251
252 kpmd = &kp->kp_md;
253 kpmd->emulate = false;
254
255 if (((instr >> 24) & 0x1f) == 0b10000)
256 kpmd->emulate = true; /* adr/adrp */
257 else if (((instr >> 26) & 0x3f) == 0b000101)
258 kpmd->emulate = true; /* b */
259 else if (((instr >> 24) & 0xff) == 0b01010100)
260 kpmd->emulate = true; /* b.cond */
261 else if (((instr >> 26) & 0x3f) == 0b100101)
262 kpmd->emulate = true; /* bl */
263 else if (((instr >> 25) & 0x3f) == 0b011010)
264 kpmd->emulate = true; /* cbnz/cbz */
265 else if (((instr >> 25) & 0x3f) == 0b011011)
266 kpmd->emulate = true; /* tbnz/tbz */
267
268 if (!kpmd->emulate)
269 kinst_trampoline_populate(kp);
270 }
271
272 static bool
kinst_instr_ldx(kinst_patchval_t instr)273 kinst_instr_ldx(kinst_patchval_t instr)
274 {
275 if (((instr >> 22) & 0xff) == 0b00100001)
276 return (true);
277
278 return (false);
279 }
280
281 static bool
kinst_instr_stx(kinst_patchval_t instr)282 kinst_instr_stx(kinst_patchval_t instr)
283 {
284 if (((instr >> 22) & 0xff) == 0b00100000)
285 return (true);
286
287 return (false);
288 }
289
290 int
kinst_make_probe(linker_file_t lf,int symindx,linker_symval_t * symval,void * opaque)291 kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval,
292 void *opaque)
293 {
294 struct kinst_probe *kp;
295 dtrace_kinst_probedesc_t *pd;
296 const char *func;
297 kinst_patchval_t *instr, *limit, *tmp;
298 int n, off;
299 bool ldxstx_block, found;
300
301 pd = opaque;
302 func = symval->name;
303
304 if (kinst_excluded(func))
305 return (0);
306 if (strcmp(func, pd->kpd_func) != 0)
307 return (0);
308
309 instr = (kinst_patchval_t *)(symval->value);
310 limit = (kinst_patchval_t *)(symval->value + symval->size);
311 if (instr >= limit)
312 return (0);
313
314 tmp = instr;
315
316 /*
317 * Ignore any bti instruction at the start of the function
318 * we need to keep it there for any indirect branches calling
319 * the function on Armv8.5+
320 */
321 if ((*tmp & BTI_MASK) == BTI_INSTR)
322 tmp++;
323
324 /* Look for stp (pre-indexed) operation */
325 found = false;
326
327 /*
328 * If the first instruction is a nop it's a specially marked
329 * asm function. We only support a nop first as it's not a normal
330 * part of the function prologue.
331 */
332 if (*tmp == NOP_INSTR)
333 found = true;
334 for (; !found && tmp < limit; tmp++) {
335 /*
336 * Functions start with "stp xt1, xt2, [xn, <const>]!" or
337 * "sub sp, sp, <const>".
338 *
339 * Sometimes the compiler will have a sub instruction that is
340 * not of the above type so don't stop if we see one.
341 */
342 if ((*tmp & LDP_STP_MASK) == STP_64) {
343 /*
344 * Assume any other store of this type means we are
345 * past the function prolog.
346 */
347 if (((*tmp >> ADDR_SHIFT) & ADDR_MASK) == 31)
348 found = true;
349 } else if ((*tmp & SUB_MASK) == SUB_INSTR &&
350 ((*tmp >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 &&
351 ((*tmp >> SUB_RN_SHIFT) & SUB_R_MASK) == 31)
352 found = true;
353 }
354
355 if (!found)
356 return (0);
357
358 ldxstx_block = false;
359 for (n = 0; instr < limit; instr++) {
360 off = (int)((uint8_t *)instr - (uint8_t *)symval->value);
361
362 /*
363 * Skip LDX/STX blocks that contain atomic operations. If a
364 * breakpoint is placed in a LDX/STX block, we violate the
365 * operation and the loop might fail.
366 */
367 if (kinst_instr_ldx(*instr))
368 ldxstx_block = true;
369 else if (kinst_instr_stx(*instr)) {
370 ldxstx_block = false;
371 continue;
372 }
373 if (ldxstx_block)
374 continue;
375
376 /*
377 * XXX: Skip ADR and ADRP instructions. The arm64 exception
378 * handler has a micro-optimization where it doesn't restore
379 * callee-saved registers when returning from exceptions in
380 * EL1. This results in a panic when the kinst emulation code
381 * modifies one of those registers.
382 */
383 if (((*instr >> 24) & 0x1f) == 0b10000)
384 continue;
385
386 if (pd->kpd_off != -1 && off != pd->kpd_off)
387 continue;
388
389 /*
390 * Prevent separate dtrace(1) instances from creating copies of
391 * the same probe.
392 */
393 LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) {
394 if (strcmp(kp->kp_func, func) == 0 &&
395 strtol(kp->kp_name, NULL, 10) == off)
396 return (0);
397 }
398 if (++n > KINST_PROBETAB_MAX) {
399 KINST_LOG("probe list full: %d entries", n);
400 return (ENOMEM);
401 }
402 kp = malloc(sizeof(struct kinst_probe), M_KINST,
403 M_WAITOK | M_ZERO);
404 kp->kp_func = func;
405 snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off);
406 kp->kp_patchpoint = instr;
407 kp->kp_savedval = *instr;
408 kp->kp_patchval = KINST_PATCHVAL;
409 if ((kp->kp_tramp = kinst_trampoline_alloc(M_WAITOK)) == NULL) {
410 KINST_LOG("cannot allocate trampoline for %p", instr);
411 return (ENOMEM);
412 }
413
414 kinst_instr_dissect(kp);
415 kinst_probe_create(kp, lf);
416 }
417 if (ldxstx_block)
418 KINST_LOG("warning: unterminated LDX/STX block");
419
420 return (0);
421 }
422
423 int
kinst_md_init(void)424 kinst_md_init(void)
425 {
426 struct kinst_cpu_state *ks;
427 int cpu;
428
429 CPU_FOREACH(cpu) {
430 ks = DPCPU_PTR(kinst_state);
431 ks->state = KINST_PROBE_ARMED;
432 }
433
434 return (0);
435 }
436
437 void
kinst_md_deinit(void)438 kinst_md_deinit(void)
439 {
440 }
441
442 /*
443 * Exclude machine-dependent functions that are not safe-to-trace.
444 */
445 bool
kinst_md_excluded(const char * name)446 kinst_md_excluded(const char *name)
447 {
448 if (strcmp(name, "handle_el1h_sync") == 0 ||
449 strcmp(name, "do_el1h_sync") == 0)
450 return (true);
451
452 return (false);
453 }
454