1 /* 2 * SPDX-License-Identifier: CDDL 1.0 3 * 4 * Copyright (c) 2023 The FreeBSD Foundation 5 * 6 * This software was developed by Christos Margiolis <christos@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 */ 9 10 #include <sys/param.h> 11 12 #include <sys/dtrace.h> 13 #include <cddl/dev/dtrace/dtrace_cddl.h> 14 15 #include "kinst.h" 16 17 /* 18 * Per-CPU trampolines used when the interrupted thread is executing with 19 * interrupts disabled. If an interrupt is raised while executing a trampoline, 20 * the interrupt thread cannot safely overwrite its trampoline if it hits a 21 * kinst probe while executing the interrupt handler. 22 */ 23 DPCPU_DEFINE_STATIC(uint8_t *, intr_tramp); 24 25 /* 26 * The double-breakpoint mechanism needs to save the current probe for the next 27 * call to kinst_invop(). As with per-CPU trampolines, this also has to be done 28 * per-CPU when interrupts are disabled. 29 */ 30 DPCPU_DEFINE_STATIC(struct kinst_probe *, intr_probe); 31 32 #define _MATCH_REG(reg) \ 33 (offsetof(struct trapframe, tf_ ## reg) / sizeof(register_t)) 34 35 static int 36 kinst_regoff(struct trapframe *frame, int n) 37 { 38 switch (n) { 39 case 0: 40 /* There is no zero register in the trapframe structure. */ 41 return (-1); 42 case 1: 43 return (_MATCH_REG(ra)); 44 case 2: 45 return (_MATCH_REG(sp)); 46 case 3: 47 return (_MATCH_REG(gp)); 48 case 4: 49 return (_MATCH_REG(tp)); 50 case 5 ... 7: 51 return (_MATCH_REG(t[n - 5])); 52 case 8 ... 9: 53 return (_MATCH_REG(s[n - 8])); 54 case 10 ... 17: 55 return (_MATCH_REG(a[n - 10])); 56 case 18 ... 27: 57 return (_MATCH_REG(s[n - 18 + 2])); 58 case 28 ... 31: 59 return (_MATCH_REG(t[n - 28 + 3])); 60 default: 61 panic("%s: unhandled register index %d", __func__, n); 62 } 63 } 64 65 static int 66 kinst_c_regoff(struct trapframe *frame, int n) 67 { 68 switch (n) { 69 case 0 ... 1: 70 return (_MATCH_REG(s[n])); 71 case 2 ... 7: 72 return (_MATCH_REG(a[n - 2])); 73 default: 74 panic("%s: unhandled register index %d", __func__, n); 75 } 76 } 77 78 #undef _MATCH_REG 79 80 static int 81 kinst_emulate(struct trapframe *frame, struct kinst_probe *kp) 82 { 83 kinst_patchval_t instr = kp->kp_savedval; 84 register_t prevpc; 85 uint64_t imm; 86 uint16_t off; 87 uint8_t funct; 88 89 if (kp->kp_md.instlen == INSN_SIZE) { 90 #define rs1_index ((instr & RS1_MASK) >> RS1_SHIFT) 91 #define rs2_index ((instr & RS2_MASK) >> RS2_SHIFT) 92 #define rd_index ((instr & RD_MASK) >> RD_SHIFT) 93 #define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] 94 #define rs2 ((register_t *)frame)[kinst_regoff(frame, rs2_index)] 95 #define rd ((register_t *)frame)[kinst_regoff(frame, rd_index)] 96 #define rs1_lval (rs1_index != 0 ? rs1 : 0) 97 #define rs2_lval (rs2_index != 0 ? rs2 : 0) 98 switch (instr & 0x7f) { 99 case 0b1101111: /* jal */ 100 imm = 0; 101 imm |= ((instr >> 21) & 0x03ff) << 1; 102 imm |= ((instr >> 20) & 0x0001) << 11; 103 imm |= ((instr >> 12) & 0x00ff) << 12; 104 imm |= ((instr >> 31) & 0x0001) << 20; 105 if (imm & 0x0000000000100000) 106 imm |= 0xfffffffffff00000; 107 if (rd_index != 0) 108 rd = frame->tf_sepc + INSN_SIZE; 109 frame->tf_sepc += imm; 110 break; 111 case 0b1100111: /* jalr */ 112 prevpc = frame->tf_sepc; 113 imm = (instr & IMM_MASK) >> IMM_SHIFT; 114 if (imm & 0x0000000000000800) 115 imm |= 0xfffffffffffff000; 116 frame->tf_sepc = (rs1_lval + imm) & ~1; 117 if (rd_index != 0) 118 rd = prevpc + INSN_SIZE; 119 break; 120 case 0b1100011: /* branch */ 121 imm = 0; 122 imm |= ((instr >> 8) & 0x000f) << 1; 123 imm |= ((instr >> 25) & 0x003f) << 5; 124 imm |= ((instr >> 7) & 0x0001) << 11; 125 imm |= ((instr >> 31) & 0x0001) << 12; 126 if (imm & 0x0000000000001000) 127 imm |= 0xfffffffffffff000; 128 funct = (instr >> 12) & 0x07; 129 switch (funct) { 130 case 0b000: /* beq */ 131 if (rs1_lval == rs2_lval) 132 frame->tf_sepc += imm; 133 else 134 frame->tf_sepc += INSN_SIZE; 135 break; 136 case 0b001: /* bne */ 137 if (rs1_lval != rs2_lval) 138 frame->tf_sepc += imm; 139 else 140 frame->tf_sepc += INSN_SIZE; 141 break; 142 case 0b100: /* blt */ 143 if ((int64_t)rs1_lval < (int64_t)rs2_lval) 144 frame->tf_sepc += imm; 145 else 146 frame->tf_sepc += INSN_SIZE; 147 break; 148 case 0b110: /* bltu */ 149 if ((uint64_t)rs1_lval < (uint64_t)rs2_lval) 150 frame->tf_sepc += imm; 151 else 152 frame->tf_sepc += INSN_SIZE; 153 break; 154 case 0b101: /* bge */ 155 if ((int64_t)rs1_lval >= (int64_t)rs2_lval) 156 frame->tf_sepc += imm; 157 else 158 frame->tf_sepc += INSN_SIZE; 159 break; 160 case 0b111: /* bgeu */ 161 if ((uint64_t)rs1_lval >= (uint64_t)rs2_lval) 162 frame->tf_sepc += imm; 163 else 164 frame->tf_sepc += INSN_SIZE; 165 break; 166 } 167 break; 168 case 0b0010111: /* auipc */ 169 imm = instr & 0xfffff000; 170 rd = frame->tf_sepc + 171 (imm & 0x0000000080000000 ? 172 imm | 0xffffffff80000000 : imm); 173 frame->tf_sepc += INSN_SIZE; 174 break; 175 } 176 #undef rs1_lval 177 #undef rs2_lval 178 #undef rs1 179 #undef rs2 180 #undef rd 181 #undef rs1_index 182 #undef rs2_index 183 #undef rd_index 184 } else { 185 switch (instr & 0x03) { 186 #define rs1 \ 187 ((register_t *)frame)[kinst_c_regoff(frame, (instr >> 7) & 0x07)] 188 case 0b01: 189 funct = (instr >> 13) & 0x07; 190 switch (funct) { 191 case 0b101: /* c.j */ 192 off = (instr >> 2) & 0x07ff; 193 imm = 0; 194 imm |= ((off >> 1) & 0x07) << 1; 195 imm |= ((off >> 9) & 0x01) << 4; 196 imm |= ((off >> 0) & 0x01) << 5; 197 imm |= ((off >> 5) & 0x01) << 6; 198 imm |= ((off >> 4) & 0x01) << 7; 199 imm |= ((off >> 7) & 0x03) << 8; 200 imm |= ((off >> 6) & 0x01) << 10; 201 imm |= ((off >> 10) & 0x01) << 11; 202 if (imm & 0x0000000000000800) 203 imm |= 0xfffffffffffff000; 204 frame->tf_sepc += imm; 205 break; 206 case 0b110: /* c.beqz */ 207 case 0b111: /* c.bnez */ 208 imm = 0; 209 imm |= ((instr >> 3) & 0x03) << 1; 210 imm |= ((instr >> 10) & 0x03) << 3; 211 imm |= ((instr >> 2) & 0x01) << 5; 212 imm |= ((instr >> 5) & 0x03) << 6; 213 imm |= ((instr >> 12) & 0x01) << 8; 214 if (imm & 0x0000000000000100) 215 imm |= 0xffffffffffffff00; 216 if (funct == 0b110 && rs1 == 0) 217 frame->tf_sepc += imm; 218 else if (funct == 0b111 && rs1 != 0) 219 frame->tf_sepc += imm; 220 else 221 frame->tf_sepc += INSN_C_SIZE; 222 break; 223 } 224 break; 225 #undef rs1 226 #define rs1_index ((instr & RD_MASK) >> RD_SHIFT) 227 #define rs1 ((register_t *)frame)[kinst_regoff(frame, rs1_index)] 228 case 0b10: 229 funct = (instr >> 13) & 0x07; 230 if (funct == 0b100 && rs1_index != 0) { 231 /* c.jr/c.jalr */ 232 prevpc = frame->tf_sepc; 233 frame->tf_sepc = rs1; 234 if (((instr >> 12) & 0x01) != 0) 235 frame->tf_ra = prevpc + INSN_C_SIZE; 236 } 237 break; 238 #undef rs1 239 #undef rs1_index 240 } 241 } 242 243 return (MATCH_C_NOP); 244 } 245 246 static int 247 kinst_jump_next_instr(struct trapframe *frame, struct kinst_probe *kp) 248 { 249 frame->tf_sepc = (register_t)((uint8_t *)kp->kp_patchpoint + 250 kp->kp_md.instlen); 251 252 return (MATCH_C_NOP); 253 } 254 255 static void 256 kinst_trampoline_populate(struct kinst_probe *kp, uint8_t *tramp) 257 { 258 static uint16_t nop = MATCH_C_NOP; 259 static uint32_t ebreak = MATCH_EBREAK; 260 int ilen; 261 262 ilen = kp->kp_md.instlen; 263 kinst_memcpy(tramp, &kp->kp_savedval, ilen); 264 265 /* 266 * Since we cannot encode large displacements in a single instruction 267 * in order to encode a far-jump back to the next instruction, and we 268 * also cannot clobber a register inside the trampoline, we execute a 269 * breakpoint after the copied instruction. kinst_invop() is 270 * responsible for detecting this special case and performing the 271 * "jump" manually. 272 * 273 * Add a NOP after a compressed instruction for padding. 274 */ 275 if (ilen == INSN_C_SIZE) 276 kinst_memcpy(&tramp[ilen], &nop, INSN_C_SIZE); 277 278 kinst_memcpy(&tramp[INSN_SIZE], &ebreak, INSN_SIZE); 279 280 fence_i(); 281 } 282 283 /* 284 * There are two ways by which an instruction is traced: 285 * 286 * - By using the trampoline. 287 * - By emulating it in software (see kinst_emulate()). 288 * 289 * The trampoline is used for instructions that can be copied and executed 290 * as-is without additional modification. However, instructions that use 291 * PC-relative addressing have to be emulated, because RISC-V doesn't allow 292 * encoding of large displacements in a single instruction, and since we cannot 293 * clobber a register in order to encode the two-instruction sequence needed to 294 * create large displacements, we cannot use the trampoline at all. 295 * Fortunately, the instructions are simple enough to be emulated in just a few 296 * lines of code. 297 * 298 * The problem discussed above also means that, unlike amd64, we cannot encode 299 * a far-jump back from the trampoline to the next instruction. The mechanism 300 * employed to achieve this functionality, is to use a breakpoint instead of a 301 * jump after the copied instruction. This breakpoint is detected and handled 302 * by kinst_invop(), which performs the jump back to the next instruction 303 * manually (see kinst_jump_next_instr()). 304 */ 305 int 306 kinst_invop(uintptr_t addr, struct trapframe *frame, uintptr_t scratch) 307 { 308 solaris_cpu_t *cpu; 309 struct kinst_probe *kp; 310 uint8_t *tramp; 311 312 /* 313 * Use per-CPU trampolines and probes if the thread executing the 314 * instruction was executing with interrupts disabled. 315 */ 316 if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) { 317 tramp = DPCPU_GET(intr_tramp); 318 kp = DPCPU_GET(intr_probe); 319 } else { 320 tramp = curthread->t_kinst_tramp; 321 kp = curthread->t_kinst_curprobe; 322 } 323 324 /* 325 * Detect if the breakpoint was triggered by the trampoline, and 326 * manually set the PC to the next instruction. 327 */ 328 if (addr == (uintptr_t)(tramp + INSN_SIZE)) 329 return (kinst_jump_next_instr(frame, kp)); 330 331 LIST_FOREACH(kp, KINST_GETPROBE(addr), kp_hashnext) { 332 if ((uintptr_t)kp->kp_patchpoint == addr) 333 break; 334 } 335 if (kp == NULL) 336 return (0); 337 338 cpu = &solaris_cpu[curcpu]; 339 cpu->cpu_dtrace_caller = addr; 340 dtrace_probe(kp->kp_id, 0, 0, 0, 0, 0); 341 cpu->cpu_dtrace_caller = 0; 342 343 if (kp->kp_md.emulate) 344 return (kinst_emulate(frame, kp)); 345 346 if (tramp == NULL) { 347 /* 348 * A trampoline allocation failed, so this probe is 349 * effectively disabled. Restore the original 350 * instruction. 351 * 352 * We can't safely print anything here, but the 353 * trampoline allocator should have left a breadcrumb in 354 * the dmesg. 355 */ 356 kinst_patch_tracepoint(kp, kp->kp_savedval); 357 frame->tf_sepc = (register_t)kp->kp_patchpoint; 358 } else { 359 kinst_trampoline_populate(kp, tramp); 360 frame->tf_sepc = (register_t)tramp; 361 if ((frame->tf_sstatus & SSTATUS_SPIE) == 0) 362 DPCPU_SET(intr_probe, kp); 363 else 364 curthread->t_kinst_curprobe = kp; 365 } 366 367 return (MATCH_C_NOP); 368 } 369 370 void 371 kinst_patch_tracepoint(struct kinst_probe *kp, kinst_patchval_t val) 372 { 373 switch (kp->kp_patchval) { 374 case KINST_C_PATCHVAL: 375 *(uint16_t *)kp->kp_patchpoint = (uint16_t)val; 376 fence_i(); 377 break; 378 case KINST_PATCHVAL: 379 *kp->kp_patchpoint = val; 380 fence_i(); 381 break; 382 } 383 } 384 385 static void 386 kinst_instr_dissect(struct kinst_probe *kp, int instrsize) 387 { 388 struct kinst_probe_md *kpmd; 389 kinst_patchval_t instr = kp->kp_savedval; 390 uint8_t funct; 391 392 kpmd = &kp->kp_md; 393 kpmd->instlen = instrsize; 394 kpmd->emulate = false; 395 396 /* 397 * The following instructions use PC-relative addressing and need to be 398 * emulated in software. 399 */ 400 if (kpmd->instlen == INSN_SIZE) { 401 switch (instr & 0x7f) { 402 case 0b1101111: /* jal */ 403 case 0b1100111: /* jalr */ 404 case 0b1100011: /* branch */ 405 case 0b0010111: /* auipc */ 406 kpmd->emulate = true; 407 break; 408 } 409 } else { 410 switch (instr & 0x03) { 411 case 0b01: 412 funct = (instr >> 13) & 0x07; 413 switch (funct) { 414 case 0b101: /* c.j */ 415 case 0b110: /* c.beqz */ 416 case 0b111: /* c.bnez */ 417 kpmd->emulate = true; 418 break; 419 } 420 break; 421 case 0b10: 422 funct = (instr >> 13) & 0x07; 423 if (funct == 0b100 && 424 ((instr >> 7) & 0x1f) != 0 && 425 ((instr >> 2) & 0x1f) == 0) 426 kpmd->emulate = true; /* c.jr/c.jalr */ 427 break; 428 } 429 } 430 } 431 432 static bool 433 kinst_instr_system(kinst_patchval_t instr) 434 { 435 if (dtrace_match_opcode(instr, MATCH_C_EBREAK, MASK_C_EBREAK) || 436 (instr & 0x7f) == 0b1110011) 437 return (true); 438 439 return (false); 440 } 441 442 static bool 443 kinst_instr_lr(kinst_patchval_t instr) 444 { 445 if (dtrace_match_opcode(instr, MATCH_LR_W, MASK_LR_W) || 446 dtrace_match_opcode(instr, MATCH_LR_D, MASK_LR_D)) 447 return (true); 448 449 return (false); 450 } 451 452 static bool 453 kinst_instr_sc(kinst_patchval_t instr) 454 { 455 if (dtrace_match_opcode(instr, MATCH_SC_W, MASK_SC_W) || 456 dtrace_match_opcode(instr, MATCH_SC_D, MASK_SC_D)) 457 return (true); 458 459 return (false); 460 } 461 462 int 463 kinst_make_probe(linker_file_t lf, int symindx, linker_symval_t *symval, 464 void *opaque) 465 { 466 struct kinst_probe *kp; 467 dtrace_kinst_probedesc_t *pd; 468 const char *func; 469 kinst_patchval_t *insn, v; 470 uint8_t *instr, *limit; 471 int instrsize, n, off; 472 bool lrsc_block, store_found, ret_found; 473 474 pd = opaque; 475 func = symval->name; 476 477 if (kinst_excluded(func)) 478 return (0); 479 if (strcmp(func, pd->kpd_func) != 0) 480 return (0); 481 482 instr = (uint8_t *)(symval->value); 483 limit = (uint8_t *)(symval->value + symval->size); 484 if (instr >= limit) 485 return (0); 486 487 /* Check for the usual function prologue. */ 488 for (insn = (kinst_patchval_t *)instr; 489 insn < (kinst_patchval_t *)limit; insn++) { 490 if (dtrace_instr_sdsp(&insn) || dtrace_instr_c_sdsp(&insn)) 491 store_found = true; 492 else if (dtrace_instr_ret(&insn) || dtrace_instr_c_ret(&insn)) 493 ret_found = true; 494 if (store_found && ret_found) 495 break; 496 } 497 if (!store_found || !ret_found) 498 return (0); 499 500 n = 0; 501 lrsc_block = false; 502 while (instr < limit) { 503 instrsize = dtrace_instr_size(instr); 504 off = (int)(instr - (uint8_t *)symval->value); 505 506 /* 507 * Avoid undefined behavior (i.e simply casting `*instr` to 508 * `kinst_patchval_t`) in case the pointer is unaligned. 509 * memcpy() can safely operate on unaligned pointers. 510 */ 511 memcpy(&v, instr, sizeof(kinst_patchval_t)); 512 513 /* Skip SYSTEM instructions. */ 514 if (kinst_instr_system(v)) 515 goto cont; 516 517 /* 518 * Skip LR/SC blocks used to build atomic operations. If a 519 * breakpoint is placed in a LR/SC block, the loop becomes 520 * unconstrained. In this case we violate the operation and the 521 * loop might fail on some implementations (see section 8.3 of 522 * the RISC-V unprivileged spec). 523 */ 524 if (kinst_instr_lr(v)) 525 lrsc_block = true; 526 else if (kinst_instr_sc(v)) { 527 lrsc_block = false; 528 goto cont; 529 } 530 if (lrsc_block) 531 goto cont; 532 533 if (pd->kpd_off != -1 && off != pd->kpd_off) 534 goto cont; 535 536 /* 537 * Prevent separate dtrace(1) instances from creating copies of 538 * the same probe. 539 */ 540 LIST_FOREACH(kp, KINST_GETPROBE(instr), kp_hashnext) { 541 if (strcmp(kp->kp_func, func) == 0 && 542 strtol(kp->kp_name, NULL, 10) == off) 543 return (0); 544 } 545 if (++n > KINST_PROBETAB_MAX) { 546 KINST_LOG("probe list full: %d entries", n); 547 return (ENOMEM); 548 } 549 kp = malloc(sizeof(struct kinst_probe), M_KINST, 550 M_WAITOK | M_ZERO); 551 kp->kp_func = func; 552 snprintf(kp->kp_name, sizeof(kp->kp_name), "%d", off); 553 kp->kp_patchpoint = (kinst_patchval_t *)instr; 554 kp->kp_savedval = v; 555 if (instrsize == INSN_SIZE) 556 kp->kp_patchval = KINST_PATCHVAL; 557 else 558 kp->kp_patchval = KINST_C_PATCHVAL; 559 560 kinst_instr_dissect(kp, instrsize); 561 kinst_probe_create(kp, lf); 562 cont: 563 instr += instrsize; 564 } 565 if (lrsc_block) 566 KINST_LOG("warning: unterminated LR/SC block"); 567 568 return (0); 569 } 570 571 int 572 kinst_md_init(void) 573 { 574 uint8_t *tramp; 575 int cpu; 576 577 CPU_FOREACH(cpu) { 578 tramp = kinst_trampoline_alloc(M_WAITOK); 579 if (tramp == NULL) 580 return (ENOMEM); 581 DPCPU_ID_SET(cpu, intr_tramp, tramp); 582 } 583 584 return (0); 585 } 586 587 void 588 kinst_md_deinit(void) 589 { 590 uint8_t *tramp; 591 int cpu; 592 593 CPU_FOREACH(cpu) { 594 tramp = DPCPU_ID_GET(cpu, intr_tramp); 595 if (tramp != NULL) { 596 kinst_trampoline_dealloc(tramp); 597 DPCPU_ID_SET(cpu, intr_tramp, NULL); 598 } 599 } 600 } 601 602 /* 603 * Exclude machine-dependent functions that are not safe-to-trace. 604 */ 605 bool 606 kinst_md_excluded(const char *name) 607 { 608 if (strcmp(name, "cpu_exception_handler") == 0 || 609 strcmp(name, "cpu_exception_handler_supervisor") == 0 || 610 strcmp(name, "cpu_exception_handler_user") == 0 || 611 strcmp(name, "do_trap_supervisor") == 0 || 612 strcmp(name, "do_trap_user") == 0) 613 return (true); 614 615 return (false); 616 } 617