1 // SPDX-License-Identifier: GPL-2.0 2 #include <ctype.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <assert.h> 7 #include <errno.h> 8 #include <fcntl.h> 9 #include <poll.h> 10 #include <pthread.h> 11 #include <unistd.h> 12 #include <linux/perf_event.h> 13 #include <linux/fs.h> 14 #include <sys/ioctl.h> 15 #include <sys/mman.h> 16 #include "trace_helpers.h" 17 #include <linux/limits.h> 18 #include <libelf.h> 19 #include <gelf.h> 20 #include "bpf/libbpf_internal.h" 21 22 #define TRACEFS_PIPE "/sys/kernel/tracing/trace_pipe" 23 #define DEBUGFS_PIPE "/sys/kernel/debug/tracing/trace_pipe" 24 25 struct ksyms { 26 struct ksym *syms; 27 size_t sym_cap; 28 size_t sym_cnt; 29 }; 30 31 static struct ksyms *ksyms; 32 static pthread_mutex_t ksyms_mutex = PTHREAD_MUTEX_INITIALIZER; 33 34 static int ksyms__add_symbol(struct ksyms *ksyms, const char *name, 35 unsigned long addr) 36 { 37 void *tmp; 38 39 tmp = strdup(name); 40 if (!tmp) 41 return -ENOMEM; 42 ksyms->syms[ksyms->sym_cnt].addr = addr; 43 ksyms->syms[ksyms->sym_cnt].name = tmp; 44 ksyms->sym_cnt++; 45 return 0; 46 } 47 48 void free_kallsyms_local(struct ksyms *ksyms) 49 { 50 unsigned int i; 51 52 if (!ksyms) 53 return; 54 55 if (!ksyms->syms) { 56 free(ksyms); 57 return; 58 } 59 60 for (i = 0; i < ksyms->sym_cnt; i++) 61 free(ksyms->syms[i].name); 62 free(ksyms->syms); 63 free(ksyms); 64 } 65 66 static struct ksyms *load_kallsyms_local_common(ksym_cmp_t cmp_cb) 67 { 68 FILE *f; 69 char func[256], buf[256]; 70 char symbol; 71 void *addr; 72 int ret; 73 struct ksyms *ksyms; 74 75 f = fopen("/proc/kallsyms", "r"); 76 if (!f) 77 return NULL; 78 79 ksyms = calloc(1, sizeof(struct ksyms)); 80 if (!ksyms) { 81 fclose(f); 82 return NULL; 83 } 84 85 while (fgets(buf, sizeof(buf), f)) { 86 if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) 87 break; 88 if (!addr) 89 continue; 90 91 ret = libbpf_ensure_mem((void **) &ksyms->syms, &ksyms->sym_cap, 92 sizeof(struct ksym), ksyms->sym_cnt + 1); 93 if (ret) 94 goto error; 95 ret = ksyms__add_symbol(ksyms, func, (unsigned long)addr); 96 if (ret) 97 goto error; 98 } 99 fclose(f); 100 qsort(ksyms->syms, ksyms->sym_cnt, sizeof(struct ksym), cmp_cb); 101 return ksyms; 102 103 error: 104 fclose(f); 105 free_kallsyms_local(ksyms); 106 return NULL; 107 } 108 109 static int ksym_cmp(const void *p1, const void *p2) 110 { 111 return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; 112 } 113 114 struct ksyms *load_kallsyms_local(void) 115 { 116 return load_kallsyms_local_common(ksym_cmp); 117 } 118 119 struct ksyms *load_kallsyms_custom_local(ksym_cmp_t cmp_cb) 120 { 121 return load_kallsyms_local_common(cmp_cb); 122 } 123 124 int load_kallsyms(void) 125 { 126 pthread_mutex_lock(&ksyms_mutex); 127 if (!ksyms) 128 ksyms = load_kallsyms_local(); 129 pthread_mutex_unlock(&ksyms_mutex); 130 return ksyms ? 0 : 1; 131 } 132 133 struct ksym *ksym_search_local(struct ksyms *ksyms, long key) 134 { 135 int start = 0, end = ksyms->sym_cnt; 136 int result; 137 138 /* kallsyms not loaded. return NULL */ 139 if (ksyms->sym_cnt <= 0) 140 return NULL; 141 142 while (start < end) { 143 size_t mid = start + (end - start) / 2; 144 145 result = key - ksyms->syms[mid].addr; 146 if (result < 0) 147 end = mid; 148 else if (result > 0) 149 start = mid + 1; 150 else 151 return &ksyms->syms[mid]; 152 } 153 154 if (start >= 1 && ksyms->syms[start - 1].addr < key && 155 key < ksyms->syms[start].addr) 156 /* valid ksym */ 157 return &ksyms->syms[start - 1]; 158 159 /* out of range. return _stext */ 160 return &ksyms->syms[0]; 161 } 162 163 struct ksym *search_kallsyms_custom_local(struct ksyms *ksyms, const void *p, 164 ksym_search_cmp_t cmp_cb) 165 { 166 int start = 0, mid, end = ksyms->sym_cnt; 167 struct ksym *ks; 168 int result; 169 170 while (start < end) { 171 mid = start + (end - start) / 2; 172 ks = &ksyms->syms[mid]; 173 result = cmp_cb(p, ks); 174 if (result < 0) 175 end = mid; 176 else if (result > 0) 177 start = mid + 1; 178 else 179 return ks; 180 } 181 182 return NULL; 183 } 184 185 struct ksym *ksym_search(long key) 186 { 187 if (!ksyms) 188 return NULL; 189 return ksym_search_local(ksyms, key); 190 } 191 192 long ksym_get_addr_local(struct ksyms *ksyms, const char *name) 193 { 194 int i; 195 196 for (i = 0; i < ksyms->sym_cnt; i++) { 197 if (strcmp(ksyms->syms[i].name, name) == 0) 198 return ksyms->syms[i].addr; 199 } 200 201 return 0; 202 } 203 204 long ksym_get_addr(const char *name) 205 { 206 if (!ksyms) 207 return 0; 208 return ksym_get_addr_local(ksyms, name); 209 } 210 211 /* open kallsyms and read symbol addresses on the fly. Without caching all symbols, 212 * this is faster than load + find. 213 */ 214 int kallsyms_find(const char *sym, unsigned long long *addr) 215 { 216 char type, name[500], *match; 217 unsigned long long value; 218 int err = 0; 219 FILE *f; 220 221 f = fopen("/proc/kallsyms", "r"); 222 if (!f) 223 return -EINVAL; 224 225 while (fscanf(f, "%llx %c %499s%*[^\n]\n", &value, &type, name) > 0) { 226 /* If CONFIG_LTO_CLANG_THIN is enabled, static variable/function 227 * symbols could be promoted to global due to cross-file inlining. 228 * For such cases, clang compiler will add .llvm.<hash> suffix 229 * to those symbols to avoid potential naming conflict. 230 * Let us ignore .llvm.<hash> suffix during symbol comparison. 231 */ 232 if (type == 'd') { 233 match = strstr(name, ".llvm."); 234 if (match) 235 *match = '\0'; 236 } 237 if (strcmp(name, sym) == 0) { 238 *addr = value; 239 goto out; 240 } 241 } 242 err = -ENOENT; 243 244 out: 245 fclose(f); 246 return err; 247 } 248 249 #ifdef PROCMAP_QUERY 250 int env_verbosity __weak = 0; 251 252 static int procmap_query(int fd, const void *addr, __u32 query_flags, size_t *start, size_t *offset, int *flags) 253 { 254 char path_buf[PATH_MAX], build_id_buf[20]; 255 struct procmap_query q; 256 int err; 257 258 memset(&q, 0, sizeof(q)); 259 q.size = sizeof(q); 260 q.query_flags = query_flags; 261 q.query_addr = (__u64)addr; 262 q.vma_name_addr = (__u64)path_buf; 263 q.vma_name_size = sizeof(path_buf); 264 q.build_id_addr = (__u64)build_id_buf; 265 q.build_id_size = sizeof(build_id_buf); 266 267 err = ioctl(fd, PROCMAP_QUERY, &q); 268 if (err < 0) { 269 err = -errno; 270 if (err == -ENOTTY) 271 return -EOPNOTSUPP; /* ioctl() not implemented yet */ 272 if (err == -ENOENT) 273 return -ESRCH; /* vma not found */ 274 return err; 275 } 276 277 if (env_verbosity >= 1) { 278 printf("VMA FOUND (addr %08lx): %08lx-%08lx %c%c%c%c %08lx %02x:%02x %ld %s (build ID: %s, %d bytes)\n", 279 (long)addr, (long)q.vma_start, (long)q.vma_end, 280 (q.vma_flags & PROCMAP_QUERY_VMA_READABLE) ? 'r' : '-', 281 (q.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) ? 'w' : '-', 282 (q.vma_flags & PROCMAP_QUERY_VMA_EXECUTABLE) ? 'x' : '-', 283 (q.vma_flags & PROCMAP_QUERY_VMA_SHARED) ? 's' : 'p', 284 (long)q.vma_offset, q.dev_major, q.dev_minor, (long)q.inode, 285 q.vma_name_size ? path_buf : "", 286 q.build_id_size ? "YES" : "NO", 287 q.build_id_size); 288 } 289 290 *start = q.vma_start; 291 *offset = q.vma_offset; 292 *flags = q.vma_flags; 293 return 0; 294 } 295 #else 296 # ifndef PROCMAP_QUERY_VMA_EXECUTABLE 297 # define PROCMAP_QUERY_VMA_EXECUTABLE 0x04 298 # endif 299 300 static int procmap_query(int fd, const void *addr, __u32 query_flags, size_t *start, size_t *offset, int *flags) 301 { 302 return -EOPNOTSUPP; 303 } 304 #endif 305 306 ssize_t get_uprobe_offset(const void *addr) 307 { 308 size_t start, base, end; 309 FILE *f; 310 char buf[256]; 311 int err, flags; 312 313 f = fopen("/proc/self/maps", "r"); 314 if (!f) 315 return -errno; 316 317 /* requested executable VMA only */ 318 err = procmap_query(fileno(f), addr, PROCMAP_QUERY_VMA_EXECUTABLE, &start, &base, &flags); 319 if (err == -EOPNOTSUPP) { 320 bool found = false; 321 322 while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &base) == 4) { 323 if (buf[2] == 'x' && (uintptr_t)addr >= start && (uintptr_t)addr < end) { 324 found = true; 325 break; 326 } 327 } 328 if (!found) { 329 fclose(f); 330 return -ESRCH; 331 } 332 } else if (err) { 333 fclose(f); 334 return err; 335 } 336 fclose(f); 337 338 #if defined(__powerpc64__) && defined(_CALL_ELF) && _CALL_ELF == 2 339 340 #define OP_RT_RA_MASK 0xffff0000UL 341 #define LIS_R2 0x3c400000UL 342 #define ADDIS_R2_R12 0x3c4c0000UL 343 #define ADDI_R2_R2 0x38420000UL 344 345 /* 346 * A PPC64 ABIv2 function may have a local and a global entry 347 * point. We need to use the local entry point when patching 348 * functions, so identify and step over the global entry point 349 * sequence. 350 * 351 * The global entry point sequence is always of the form: 352 * 353 * addis r2,r12,XXXX 354 * addi r2,r2,XXXX 355 * 356 * A linker optimisation may convert the addis to lis: 357 * 358 * lis r2,XXXX 359 * addi r2,r2,XXXX 360 */ 361 { 362 const __u32 *insn = (const __u32 *)(uintptr_t)addr; 363 364 if ((((*insn & OP_RT_RA_MASK) == ADDIS_R2_R12) || 365 ((*insn & OP_RT_RA_MASK) == LIS_R2)) && 366 ((*(insn + 1) & OP_RT_RA_MASK) == ADDI_R2_R2)) 367 return (uintptr_t)(insn + 2) - start + base; 368 } 369 #endif 370 return (uintptr_t)addr - start + base; 371 } 372 373 ssize_t get_rel_offset(uintptr_t addr) 374 { 375 size_t start, end, offset; 376 char buf[256]; 377 FILE *f; 378 int err, flags; 379 380 f = fopen("/proc/self/maps", "r"); 381 if (!f) 382 return -errno; 383 384 err = procmap_query(fileno(f), (const void *)addr, 0, &start, &offset, &flags); 385 if (err == 0) { 386 fclose(f); 387 return (size_t)addr - start + offset; 388 } else if (err != -EOPNOTSUPP) { 389 fclose(f); 390 return err; 391 } else if (err) { 392 while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &offset) == 4) { 393 if (addr >= start && addr < end) { 394 fclose(f); 395 return (size_t)addr - start + offset; 396 } 397 } 398 } 399 400 fclose(f); 401 return -EINVAL; 402 } 403 404 static int 405 parse_build_id_buf(const void *note_start, Elf32_Word note_size, char *build_id) 406 { 407 Elf32_Word note_offs = 0; 408 409 while (note_offs + sizeof(Elf32_Nhdr) < note_size) { 410 Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); 411 412 if (nhdr->n_type == 3 && nhdr->n_namesz == sizeof("GNU") && 413 !strcmp((char *)(nhdr + 1), "GNU") && nhdr->n_descsz > 0 && 414 nhdr->n_descsz <= BPF_BUILD_ID_SIZE) { 415 memcpy(build_id, note_start + note_offs + 416 ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), nhdr->n_descsz); 417 memset(build_id + nhdr->n_descsz, 0, BPF_BUILD_ID_SIZE - nhdr->n_descsz); 418 return (int) nhdr->n_descsz; 419 } 420 421 note_offs = note_offs + sizeof(Elf32_Nhdr) + 422 ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); 423 } 424 425 return -ENOENT; 426 } 427 428 /* Reads binary from *path* file and returns it in the *build_id* buffer 429 * with *size* which is expected to be at least BPF_BUILD_ID_SIZE bytes. 430 * Returns size of build id on success. On error the error value is 431 * returned. 432 */ 433 int read_build_id(const char *path, char *build_id, size_t size) 434 { 435 int fd, err = -EINVAL; 436 Elf *elf = NULL; 437 GElf_Ehdr ehdr; 438 size_t max, i; 439 440 if (size < BPF_BUILD_ID_SIZE) 441 return -EINVAL; 442 443 fd = open(path, O_RDONLY | O_CLOEXEC); 444 if (fd < 0) 445 return -errno; 446 447 (void)elf_version(EV_CURRENT); 448 449 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); 450 if (!elf) 451 goto out; 452 if (elf_kind(elf) != ELF_K_ELF) 453 goto out; 454 if (!gelf_getehdr(elf, &ehdr)) 455 goto out; 456 457 for (i = 0; i < ehdr.e_phnum; i++) { 458 GElf_Phdr mem, *phdr; 459 char *data; 460 461 phdr = gelf_getphdr(elf, i, &mem); 462 if (!phdr) 463 goto out; 464 if (phdr->p_type != PT_NOTE) 465 continue; 466 data = elf_rawfile(elf, &max); 467 if (!data) 468 goto out; 469 if (phdr->p_offset + phdr->p_memsz > max) 470 goto out; 471 err = parse_build_id_buf(data + phdr->p_offset, phdr->p_memsz, build_id); 472 if (err > 0) 473 break; 474 } 475 476 out: 477 if (elf) 478 elf_end(elf); 479 close(fd); 480 return err; 481 } 482 483 int read_trace_pipe_iter(void (*cb)(const char *str, void *data), void *data, int iter) 484 { 485 size_t buflen, n; 486 char *buf = NULL; 487 FILE *fp = NULL; 488 489 if (access(TRACEFS_PIPE, F_OK) == 0) 490 fp = fopen(TRACEFS_PIPE, "r"); 491 else 492 fp = fopen(DEBUGFS_PIPE, "r"); 493 if (!fp) 494 return -1; 495 496 /* We do not want to wait forever when iter is specified. */ 497 if (iter) 498 fcntl(fileno(fp), F_SETFL, O_NONBLOCK); 499 500 while ((n = getline(&buf, &buflen, fp) >= 0) || errno == EAGAIN) { 501 if (n > 0) 502 cb(buf, data); 503 if (iter && !(--iter)) 504 break; 505 } 506 507 free(buf); 508 if (fp) 509 fclose(fp); 510 return 0; 511 } 512 513 static void trace_pipe_cb(const char *str, void *data) 514 { 515 printf("%s", str); 516 } 517 518 void read_trace_pipe(void) 519 { 520 read_trace_pipe_iter(trace_pipe_cb, NULL, 0); 521 } 522