1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright 2022 Christos Margiolis <christos@FreeBSD.org> 5 * Copyright 2022 Mark Johnston <markj@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 10 #include <machine/cpufunc.h> 11 #include <machine/md_var.h> 12 13 #include <sys/dtrace.h> 14 #include <cddl/dev/dtrace/dtrace_cddl.h> 15 #include <dis_tables.h> 16 17 #include "kinst.h" 18 19 #define KINST_PUSHL_RBP 0x55 20 #define KINST_STI 0xfb 21 #define KINST_POPF 0x9d 22 23 #define KINST_MODRM_MOD(b) (((b) & 0xc0) >> 6) 24 #define KINST_MODRM_REG(b) (((b) & 0x38) >> 3) 25 #define KINST_MODRM_RM(b) ((b) & 0x07) 26 27 #define KINST_SIB_SCALE(s) (((s) & 0xc0) >> 6) 28 #define KINST_SIB_INDEX(s) (((s) & 0x38) >> 3) 29 #define KINST_SIB_BASE(s) (((s) & 0x07) >> 0) 30 31 #define KINST_REX_W(r) (((r) & 0x08) >> 3) 32 #define KINST_REX_R(r) (((r) & 0x04) >> 2) 33 #define KINST_REX_X(r) (((r) & 0x02) >> 1) 34 #define KINST_REX_B(r) (((r) & 0x01) >> 0) 35 36 #define KINST_F_CALL 0x0001 /* instruction is a "call" */ 37 #define KINST_F_DIRECT_CALL 0x0002 /* instruction is a direct call */ 38 #define KINST_F_RIPREL 0x0004 /* instruction is position-dependent */ 39 #define KINST_F_JMP 0x0008 /* instruction is a %rip-relative jmp */ 40 #define KINST_F_MOD_DIRECT 0x0010 /* operand is not a memory address */ 41 42 /* 43 * Map ModR/M register bits to a trapframe offset. 44 */ 45 static int 46 kinst_regoff(int reg) 47 { 48 #define _MATCH_REG(i, reg) \ 49 case i: \ 50 return (offsetof(struct trapframe, tf_ ## reg) / \ 51 sizeof(register_t)) 52 switch (reg) { 53 _MATCH_REG( 0, rax); 54 _MATCH_REG( 1, rcx); 55 _MATCH_REG( 2, rdx); 56 _MATCH_REG( 3, rbx); 57 _MATCH_REG( 4, rsp); /* SIB when mod != 3 */ 58 _MATCH_REG( 5, rbp); 59 _MATCH_REG( 6, rsi); 60 _MATCH_REG( 7, rdi); 61 _MATCH_REG( 8, r8); /* REX.R is set */ 62 _MATCH_REG( 9, r9); 63 _MATCH_REG(10, r10); 64 _MATCH_REG(11, r11); 65 _MATCH_REG(12, r12); 66 _MATCH_REG(13, r13); 67 _MATCH_REG(14, r14); 68 _MATCH_REG(15, r15); 69 } 70 #undef _MATCH_REG 71 panic("%s: unhandled register index %d", __func__, reg); 72 } 73 74 /* 75 * Obtain the specified register's value. 76 */ 77 static uint64_t 78 kinst_regval(struct trapframe *frame, int reg) 79 { 80 if (reg == -1) 81 return (0); 82 return (((register_t *)frame)[kinst_regoff(reg)]); 83 } 84 85 static uint32_t 86 kinst_riprel_disp(struct kinst_probe *kp, void *dst) 87 { 88 return ((uint32_t)((intptr_t)kp->kp_patchpoint + kp->kp_md.disp - 89 (intptr_t)dst)); 90 } 91 92 static void 93 kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp) 94 { 95 uint8_t *instr; 96 uint32_t disp; 97 int ilen; 98 99 ilen = kp->kp_md.tinstlen; 100 101 memcpy(tramp, kp->kp_md.template, ilen); 102 if ((kp->kp_md.flags & KINST_F_RIPREL) != 0) { 103 disp = kinst_riprel_disp(kp, tramp); 104 memcpy(&tramp[kp->kp_md.dispoff], &disp, sizeof(uint32_t)); 105 } 106 107 /* 108 * The following position-independent jmp takes us back to the 109 * original code. It is encoded as "jmp *0(%rip)" (six bytes), 110 * followed by the absolute address of the instruction following 111 * the one that was traced (eight bytes). 112 */ 113 tramp[ilen + 0] = 0xff; 114 tramp[ilen + 1] = 0x25; 115 tramp[ilen + 2] = 0x00; 116 tramp[ilen + 3] = 0x00; 117 tramp[ilen + 4] = 0x00; 118 tramp[ilen + 5] = 0x00; 119 instr = kp->kp_patchpoint + kp->kp_md.instlen; 120 memcpy(&tramp[ilen + 6], &instr, sizeof(uintptr_t)); 121 } 122 123 int 124 kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) 125 { 126 solaris_cpu_t *cpu; 127 uintptr_t *stack, retaddr; 128 struct kinst_probe *kp; 129 struct kinst_probe_md *kpmd; 130 uint8_t *tramp; 131 132 stack = (uintptr_t *)frame->tf_rsp; 133 cpu = &solaris_cpu[curcpu]; 134 135 LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { 136 if ((uintptr_t)kp->kp_patchpoint == addr) 137 break; 138 } 139 if (kp == NULL) 140 return (0); 141 142 /* 143 * Report the address of the breakpoint for the benefit of consumers 144 * fetching register values with regs[]. 145 */ 146 frame->tf_rip--; 147 148 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 149 cpu->cpu_dtrace_caller = stack[0]; 150 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR); 151 dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); 152 cpu->cpu_dtrace_caller = 0; 153 154 kpmd = &kp->kp_md; 155 if ((kpmd->flags & KINST_F_CALL) != 0) { 156 /* 157 * dtrace_invop_start() reserves space on the stack to 158 * store the return address of the call instruction. 159 */ 160 retaddr = (uintptr_t)(kp->kp_patchpoint + kpmd->instlen); 161 *(uintptr_t *)scratch = retaddr; 162 163 if ((kpmd->flags & KINST_F_DIRECT_CALL) != 0) { 164 frame->tf_rip = (uintptr_t)(kp->kp_patchpoint + 165 kpmd->disp + kpmd->instlen); 166 } else { 167 register_t rval; 168 169 if (kpmd->reg1 == -1 && kpmd->reg2 == -1) { 170 /* rip-relative */ 171 rval = frame->tf_rip + kpmd->instlen; 172 } else { 173 /* indirect */ 174 rval = kinst_regval(frame, kpmd->reg1) + 175 (kinst_regval(frame, kpmd->reg2) << 176 kpmd->scale); 177 } 178 179 if ((kpmd->flags & KINST_F_MOD_DIRECT) != 0) { 180 frame->tf_rip = rval + kpmd->disp; 181 } else { 182 frame->tf_rip = 183 *(uintptr_t *)(rval + kpmd->disp); 184 } 185 } 186 return (DTRACE_INVOP_CALL); 187 } else { 188 tramp = curthread->t_kinst; 189 if (tramp == NULL) { 190 /* 191 * A trampoline allocation failed, so this probe is 192 * effectively disabled. Restore the original 193 * instruction. 194 * 195 * We can't safely print anything here, but the 196 * trampoline allocator should have left a breadcrumb in 197 * the dmesg. 198 */ 199 kinst_patch_tracepoint(kp, kp->kp_savedval); 200 frame->tf_rip = (register_t)kp->kp_patchpoint; 201 } else { 202 kinst_trampoline_populate(kp, tramp); 203 frame->tf_rip = (register_t)tramp; 204 } 205 return (DTRACE_INVOP_NOP); 206 } 207 } 208 209 void 210 kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) 211 { 212 register_t reg; 213 int oldwp; 214 215 reg = intr_disable(); 216 oldwp = disable_wp(); 217 *kp->kp_patchpoint = val; 218 restore_wp(oldwp); 219 intr_restore(reg); 220 } 221 222 static void 223 kinst_set_disp8(struct kinst_probe *kp, uint8_t byte) 224 { 225 kp->kp_md.disp = (int64_t)(int8_t)byte; 226 } 227 228 static void 229 kinst_set_disp32(struct kinst_probe *kp, uint8_t *bytes) 230 { 231 int32_t disp32; 232 233 memcpy(&disp32, bytes, sizeof(disp32)); 234 kp->kp_md.disp = (int64_t)disp32; 235 } 236 237 static int 238 kinst_dis_get_byte(void *p) 239 { 240 int ret; 241 uint8_t **instr = p; 242 243 ret = **instr; 244 (*instr)++; 245 246 return (ret); 247 } 248 249 /* 250 * Set up all of the state needed to faithfully execute a probed instruction. 251 * 252 * In the simple case, we copy the instruction unmodified to a per-thread 253 * trampoline, wherein it is followed by a jump back to the original code. 254 * - Instructions can have %rip as an operand: 255 * - with %rip-relative addressing encoded in ModR/M, or 256 * - implicitly as a part of the instruction definition (jmp, call). 257 * - Call instructions (which may be %rip-relative) need to push the correct 258 * return address onto the stack. 259 * 260 * Call instructions are simple enough to be emulated in software, so we simply 261 * do not use the trampoline mechanism in that case. kinst_invop() will compute 262 * the branch target using the address info computed here (register operands and 263 * displacement). 264 * 265 * %rip-relative operands encoded using the ModR/M byte always use a 32-bit 266 * displacement; when populating the trampoline the displacement is adjusted to 267 * be relative to the trampoline address. Trampolines are always allocated 268 * above KERNBASE for this reason. 269 * 270 * For other %rip-relative operands (just jumps) we take the same approach. 271 * Instructions which specify an 8-bit displacement must be rewritten to use a 272 * 32-bit displacement. 273 */ 274 static int 275 kinst_instr_dissect(struct kinst_probe *kp, uint8_t **instr) 276 { 277 struct kinst_probe_md *kpmd; 278 dis86_t d86; 279 uint8_t *bytes, modrm, rex; 280 int dispoff, i, ilen, opcidx; 281 282 kpmd = &kp->kp_md; 283 284 d86.d86_data = instr; 285 d86.d86_get_byte = kinst_dis_get_byte; 286 d86.d86_check_func = NULL; 287 if (dtrace_disx86(&d86, SIZE64) != 0) { 288 KINST_LOG("failed to disassemble instruction at: %p", *instr); 289 return (EINVAL); 290 } 291 bytes = d86.d86_bytes; 292 kpmd->instlen = kpmd->tinstlen = d86.d86_len; 293 294 /* 295 * Skip over prefixes, save REX. 296 */ 297 rex = 0; 298 for (i = 0; i < kpmd->instlen; i++) { 299 switch (bytes[i]) { 300 case 0xf0 ... 0xf3: 301 /* group 1 */ 302 continue; 303 case 0x26: 304 case 0x2e: 305 case 0x36: 306 case 0x3e: 307 case 0x64: 308 case 0x65: 309 /* group 2 */ 310 continue; 311 case 0x66: 312 /* group 3 */ 313 continue; 314 case 0x67: 315 /* group 4 */ 316 continue; 317 case 0x40 ... 0x4f: 318 /* REX */ 319 rex = bytes[i]; 320 continue; 321 } 322 break; 323 } 324 KASSERT(i < kpmd->instlen, 325 ("%s: failed to disassemble instruction at %p", __func__, bytes)); 326 opcidx = i; 327 328 /* 329 * Identify instructions of interest by opcode: calls and jumps. 330 * Extract displacements. 331 */ 332 dispoff = -1; 333 switch (bytes[opcidx]) { 334 case 0x0f: 335 switch (bytes[opcidx + 1]) { 336 case 0x80 ... 0x8f: 337 /* conditional jmp near */ 338 kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; 339 dispoff = opcidx + 2; 340 kinst_set_disp32(kp, &bytes[dispoff]); 341 break; 342 } 343 break; 344 case 0xe3: 345 /* 346 * There is no straightforward way to translate this instruction 347 * to use a 32-bit displacement. Fortunately, it is rarely 348 * used. 349 */ 350 return (EINVAL); 351 case 0x70 ... 0x7f: 352 /* conditional jmp short */ 353 kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; 354 dispoff = opcidx + 1; 355 kinst_set_disp8(kp, bytes[dispoff]); 356 break; 357 case 0xe9: 358 /* unconditional jmp near */ 359 kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; 360 dispoff = opcidx + 1; 361 kinst_set_disp32(kp, &bytes[dispoff]); 362 break; 363 case 0xeb: 364 /* unconditional jmp short */ 365 kpmd->flags |= KINST_F_JMP | KINST_F_RIPREL; 366 dispoff = opcidx + 1; 367 kinst_set_disp8(kp, bytes[dispoff]); 368 break; 369 case 0xe8: 370 case 0x9a: 371 /* direct call */ 372 kpmd->flags |= KINST_F_CALL | KINST_F_DIRECT_CALL; 373 dispoff = opcidx + 1; 374 kinst_set_disp32(kp, &bytes[dispoff]); 375 break; 376 case 0xff: 377 KASSERT(d86.d86_got_modrm, 378 ("no ModR/M byte for instr at %p", *instr - kpmd->instlen)); 379 switch (KINST_MODRM_REG(bytes[d86.d86_rmindex])) { 380 case 0x02: 381 case 0x03: 382 /* indirect call */ 383 kpmd->flags |= KINST_F_CALL; 384 break; 385 case 0x04: 386 case 0x05: 387 /* indirect jump */ 388 kpmd->flags |= KINST_F_JMP; 389 break; 390 } 391 } 392 393 /* 394 * If there's a ModR/M byte, we need to check it to see if the operand 395 * is %rip-relative, and rewrite the displacement if so. If not, we 396 * might still have to extract operand info if this is a call 397 * instruction. 398 */ 399 if (d86.d86_got_modrm) { 400 uint8_t mod, rm, sib; 401 402 kpmd->reg1 = kpmd->reg2 = -1; 403 404 modrm = bytes[d86.d86_rmindex]; 405 mod = KINST_MODRM_MOD(modrm); 406 rm = KINST_MODRM_RM(modrm); 407 if (mod == 0 && rm == 5) { 408 kpmd->flags |= KINST_F_RIPREL; 409 dispoff = d86.d86_rmindex + 1; 410 kinst_set_disp32(kp, &bytes[dispoff]); 411 } else if ((kpmd->flags & KINST_F_CALL) != 0) { 412 bool havesib; 413 414 havesib = (mod != 3 && rm == 4); 415 dispoff = d86.d86_rmindex + (havesib ? 2 : 1); 416 if (mod == 1) 417 kinst_set_disp8(kp, bytes[dispoff]); 418 else if (mod == 2) 419 kinst_set_disp32(kp, &bytes[dispoff]); 420 else if (mod == 3) 421 kpmd->flags |= KINST_F_MOD_DIRECT; 422 423 if (havesib) { 424 sib = bytes[d86.d86_rmindex + 1]; 425 if (KINST_SIB_BASE(sib) != 5) { 426 kpmd->reg1 = KINST_SIB_BASE(sib) | 427 (KINST_REX_B(rex) << 3); 428 } 429 kpmd->scale = KINST_SIB_SCALE(sib); 430 kpmd->reg2 = KINST_SIB_INDEX(sib) | 431 (KINST_REX_X(rex) << 3); 432 } else { 433 kpmd->reg1 = rm | (KINST_REX_B(rex) << 3); 434 } 435 } 436 } 437 438 /* 439 * Calls are emulated in software; once operands are decoded we have 440 * nothing else to do. 441 */ 442 if ((kpmd->flags & KINST_F_CALL) != 0) 443 return (0); 444 445 /* 446 * Allocate and populate an instruction trampoline template. 447 * 448 * Position-independent instructions can simply be copied, but 449 * position-dependent instructions require some surgery: jump 450 * instructions with an 8-bit displacement need to be converted to use a 451 * 32-bit displacement, and the adjusted displacement needs to be 452 * computed. 453 */ 454 ilen = kpmd->instlen; 455 if ((kpmd->flags & KINST_F_RIPREL) != 0) { 456 if ((kpmd->flags & KINST_F_JMP) == 0 || 457 bytes[opcidx] == 0x0f || 458 bytes[opcidx] == 0xe9 || 459 bytes[opcidx] == 0xff) { 460 memcpy(kpmd->template, bytes, dispoff); 461 memcpy(&kpmd->template[dispoff + 4], 462 &bytes[dispoff + 4], ilen - (dispoff + 4)); 463 kpmd->dispoff = dispoff; 464 } else if (bytes[opcidx] == 0xeb) { 465 memcpy(kpmd->template, bytes, opcidx); 466 kpmd->template[opcidx] = 0xe9; 467 kpmd->dispoff = opcidx + 1; 468 469 /* Instruction length changes from 2 to 5. */ 470 kpmd->tinstlen = 5; 471 kpmd->disp -= 3; 472 } else if (bytes[opcidx] >= 0x70 && bytes[opcidx] <= 0x7f) { 473 memcpy(kpmd->template, bytes, opcidx); 474 kpmd->template[opcidx] = 0x0f; 475 kpmd->template[opcidx + 1] = bytes[opcidx] + 0x10; 476 kpmd->dispoff = opcidx + 2; 477 478 /* Instruction length changes from 2 to 6. */ 479 kpmd->tinstlen = 6; 480 kpmd->disp -= 4; 481 } else { 482 panic("unhandled opcode %#x", bytes[opcidx]); 483 } 484 } else { 485 memcpy(kpmd->template, bytes, ilen); 486 } 487 488 return (0); 489 } 490 491 int 492 kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, 493 void *opaque) 494 { 495 struct kinst_probe *kp; 496 dtrace_kinst_probedesc_t *pd; 497 const char *func; 498 int error, n, off; 499 uint8_t *instr, *limit; 500 501 pd = opaque; 502 func = symval->name; 503 if (strcmp(func, pd->kpd_func) != 0 || strcmp(func, "trap_check") == 0) 504 return (0); 505 506 instr = (uint8_t *)symval->value; 507 limit = (uint8_t *)symval->value + symval->size; 508 if (instr >= limit) 509 return (0); 510 511 /* 512 * Ignore functions not beginning with the usual function prologue. 513 * These might correspond to assembly routines with which we should not 514 * meddle. 515 */ 516 if (*instr != KINST_PUSHL_RBP) 517 return (0); 518 519 n = 0; 520 while (instr < limit) { 521 off = (int)(instr - (uint8_t *)symval->value); 522 if (pd->kpd_off != -1 && off != pd->kpd_off) { 523 instr += dtrace_instr_size(instr); 524 continue; 525 } 526 527 /* 528 * Prevent separate dtrace(1) instances from creating copies of 529 * the same probe. 530 */ 531 LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { 532 if (strcmp(kp->kp_func, func) == 0 && 533 strtol(kp->kp_name, NULL, 10) == off) 534 return (0); 535 } 536 if (++n > KINST_PROBETAB_MAX) { 537 KINST_LOG("probe list full: %d entries", n); 538 return (ENOMEM); 539 } 540 kp = malloc(sizeof(struct kinst_probe), M_KINST, 541 M_WAITOK | M_ZERO); 542 kp->kp_func = func; 543 snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); 544 kp->kp_savedval = *instr; 545 kp->kp_patchval = KINST_PATCHVAL; 546 kp->kp_patchpoint = instr; 547 548 error = kinst_instr_dissect(kp, &instr); 549 if (error != 0) 550 return (error); 551 552 kinst_probe_create(kp, lf); 553 } 554 555 return (0); 556 } 557