1 // SPDX-License-Identifier: GPL-2.0 2 #include <ctype.h> 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <assert.h> 7 #include <errno.h> 8 #include <fcntl.h> 9 #include <poll.h> 10 #include <pthread.h> 11 #include <unistd.h> 12 #include <linux/perf_event.h> 13 #include <sys/mman.h> 14 #include "trace_helpers.h" 15 #include <linux/limits.h> 16 #include <libelf.h> 17 #include <gelf.h> 18 #include "bpf/libbpf_internal.h" 19 20 #define TRACEFS_PIPE "/sys/kernel/tracing/trace_pipe" 21 #define DEBUGFS_PIPE "/sys/kernel/debug/tracing/trace_pipe" 22 23 struct ksyms { 24 struct ksym *syms; 25 size_t sym_cap; 26 size_t sym_cnt; 27 }; 28 29 static struct ksyms *ksyms; 30 static pthread_mutex_t ksyms_mutex = PTHREAD_MUTEX_INITIALIZER; 31 32 static int ksyms__add_symbol(struct ksyms *ksyms, const char *name, 33 unsigned long addr) 34 { 35 void *tmp; 36 37 tmp = strdup(name); 38 if (!tmp) 39 return -ENOMEM; 40 ksyms->syms[ksyms->sym_cnt].addr = addr; 41 ksyms->syms[ksyms->sym_cnt].name = tmp; 42 ksyms->sym_cnt++; 43 return 0; 44 } 45 46 void free_kallsyms_local(struct ksyms *ksyms) 47 { 48 unsigned int i; 49 50 if (!ksyms) 51 return; 52 53 if (!ksyms->syms) { 54 free(ksyms); 55 return; 56 } 57 58 for (i = 0; i < ksyms->sym_cnt; i++) 59 free(ksyms->syms[i].name); 60 free(ksyms->syms); 61 free(ksyms); 62 } 63 64 static struct ksyms *load_kallsyms_local_common(ksym_cmp_t cmp_cb) 65 { 66 FILE *f; 67 char func[256], buf[256]; 68 char symbol; 69 void *addr; 70 int ret; 71 struct ksyms *ksyms; 72 73 f = fopen("/proc/kallsyms", "r"); 74 if (!f) 75 return NULL; 76 77 ksyms = calloc(1, sizeof(struct ksyms)); 78 if (!ksyms) { 79 fclose(f); 80 return NULL; 81 } 82 83 while (fgets(buf, sizeof(buf), f)) { 84 if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) 85 break; 86 if (!addr) 87 continue; 88 89 ret = libbpf_ensure_mem((void **) &ksyms->syms, &ksyms->sym_cap, 90 sizeof(struct ksym), ksyms->sym_cnt + 1); 91 if (ret) 92 goto error; 93 ret = ksyms__add_symbol(ksyms, func, (unsigned long)addr); 94 if (ret) 95 goto error; 96 } 97 fclose(f); 98 qsort(ksyms->syms, ksyms->sym_cnt, sizeof(struct ksym), cmp_cb); 99 return ksyms; 100 101 error: 102 fclose(f); 103 free_kallsyms_local(ksyms); 104 return NULL; 105 } 106 107 static int ksym_cmp(const void *p1, const void *p2) 108 { 109 return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; 110 } 111 112 struct ksyms *load_kallsyms_local(void) 113 { 114 return load_kallsyms_local_common(ksym_cmp); 115 } 116 117 struct ksyms *load_kallsyms_custom_local(ksym_cmp_t cmp_cb) 118 { 119 return load_kallsyms_local_common(cmp_cb); 120 } 121 122 int load_kallsyms(void) 123 { 124 pthread_mutex_lock(&ksyms_mutex); 125 if (!ksyms) 126 ksyms = load_kallsyms_local(); 127 pthread_mutex_unlock(&ksyms_mutex); 128 return ksyms ? 0 : 1; 129 } 130 131 struct ksym *ksym_search_local(struct ksyms *ksyms, long key) 132 { 133 int start = 0, end = ksyms->sym_cnt; 134 int result; 135 136 /* kallsyms not loaded. return NULL */ 137 if (ksyms->sym_cnt <= 0) 138 return NULL; 139 140 while (start < end) { 141 size_t mid = start + (end - start) / 2; 142 143 result = key - ksyms->syms[mid].addr; 144 if (result < 0) 145 end = mid; 146 else if (result > 0) 147 start = mid + 1; 148 else 149 return &ksyms->syms[mid]; 150 } 151 152 if (start >= 1 && ksyms->syms[start - 1].addr < key && 153 key < ksyms->syms[start].addr) 154 /* valid ksym */ 155 return &ksyms->syms[start - 1]; 156 157 /* out of range. return _stext */ 158 return &ksyms->syms[0]; 159 } 160 161 struct ksym *search_kallsyms_custom_local(struct ksyms *ksyms, const void *p, 162 ksym_search_cmp_t cmp_cb) 163 { 164 int start = 0, mid, end = ksyms->sym_cnt; 165 struct ksym *ks; 166 int result; 167 168 while (start < end) { 169 mid = start + (end - start) / 2; 170 ks = &ksyms->syms[mid]; 171 result = cmp_cb(p, ks); 172 if (result < 0) 173 end = mid; 174 else if (result > 0) 175 start = mid + 1; 176 else 177 return ks; 178 } 179 180 return NULL; 181 } 182 183 struct ksym *ksym_search(long key) 184 { 185 if (!ksyms) 186 return NULL; 187 return ksym_search_local(ksyms, key); 188 } 189 190 long ksym_get_addr_local(struct ksyms *ksyms, const char *name) 191 { 192 int i; 193 194 for (i = 0; i < ksyms->sym_cnt; i++) { 195 if (strcmp(ksyms->syms[i].name, name) == 0) 196 return ksyms->syms[i].addr; 197 } 198 199 return 0; 200 } 201 202 long ksym_get_addr(const char *name) 203 { 204 if (!ksyms) 205 return 0; 206 return ksym_get_addr_local(ksyms, name); 207 } 208 209 /* open kallsyms and read symbol addresses on the fly. Without caching all symbols, 210 * this is faster than load + find. 211 */ 212 int kallsyms_find(const char *sym, unsigned long long *addr) 213 { 214 char type, name[500], *match; 215 unsigned long long value; 216 int err = 0; 217 FILE *f; 218 219 f = fopen("/proc/kallsyms", "r"); 220 if (!f) 221 return -EINVAL; 222 223 while (fscanf(f, "%llx %c %499s%*[^\n]\n", &value, &type, name) > 0) { 224 /* If CONFIG_LTO_CLANG_THIN is enabled, static variable/function 225 * symbols could be promoted to global due to cross-file inlining. 226 * For such cases, clang compiler will add .llvm.<hash> suffix 227 * to those symbols to avoid potential naming conflict. 228 * Let us ignore .llvm.<hash> suffix during symbol comparison. 229 */ 230 if (type == 'd') { 231 match = strstr(name, ".llvm."); 232 if (match) 233 *match = '\0'; 234 } 235 if (strcmp(name, sym) == 0) { 236 *addr = value; 237 goto out; 238 } 239 } 240 err = -ENOENT; 241 242 out: 243 fclose(f); 244 return err; 245 } 246 247 ssize_t get_uprobe_offset(const void *addr) 248 { 249 size_t start, end, base; 250 char buf[256]; 251 bool found = false; 252 FILE *f; 253 254 f = fopen("/proc/self/maps", "r"); 255 if (!f) 256 return -errno; 257 258 while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &base) == 4) { 259 if (buf[2] == 'x' && (uintptr_t)addr >= start && (uintptr_t)addr < end) { 260 found = true; 261 break; 262 } 263 } 264 265 fclose(f); 266 267 if (!found) 268 return -ESRCH; 269 270 #if defined(__powerpc64__) && defined(_CALL_ELF) && _CALL_ELF == 2 271 272 #define OP_RT_RA_MASK 0xffff0000UL 273 #define LIS_R2 0x3c400000UL 274 #define ADDIS_R2_R12 0x3c4c0000UL 275 #define ADDI_R2_R2 0x38420000UL 276 277 /* 278 * A PPC64 ABIv2 function may have a local and a global entry 279 * point. We need to use the local entry point when patching 280 * functions, so identify and step over the global entry point 281 * sequence. 282 * 283 * The global entry point sequence is always of the form: 284 * 285 * addis r2,r12,XXXX 286 * addi r2,r2,XXXX 287 * 288 * A linker optimisation may convert the addis to lis: 289 * 290 * lis r2,XXXX 291 * addi r2,r2,XXXX 292 */ 293 { 294 const __u32 *insn = (const __u32 *)(uintptr_t)addr; 295 296 if ((((*insn & OP_RT_RA_MASK) == ADDIS_R2_R12) || 297 ((*insn & OP_RT_RA_MASK) == LIS_R2)) && 298 ((*(insn + 1) & OP_RT_RA_MASK) == ADDI_R2_R2)) 299 return (uintptr_t)(insn + 2) - start + base; 300 } 301 #endif 302 return (uintptr_t)addr - start + base; 303 } 304 305 ssize_t get_rel_offset(uintptr_t addr) 306 { 307 size_t start, end, offset; 308 char buf[256]; 309 FILE *f; 310 311 f = fopen("/proc/self/maps", "r"); 312 if (!f) 313 return -errno; 314 315 while (fscanf(f, "%zx-%zx %s %zx %*[^\n]\n", &start, &end, buf, &offset) == 4) { 316 if (addr >= start && addr < end) { 317 fclose(f); 318 return (size_t)addr - start + offset; 319 } 320 } 321 322 fclose(f); 323 return -EINVAL; 324 } 325 326 static int 327 parse_build_id_buf(const void *note_start, Elf32_Word note_size, char *build_id) 328 { 329 Elf32_Word note_offs = 0; 330 331 while (note_offs + sizeof(Elf32_Nhdr) < note_size) { 332 Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); 333 334 if (nhdr->n_type == 3 && nhdr->n_namesz == sizeof("GNU") && 335 !strcmp((char *)(nhdr + 1), "GNU") && nhdr->n_descsz > 0 && 336 nhdr->n_descsz <= BPF_BUILD_ID_SIZE) { 337 memcpy(build_id, note_start + note_offs + 338 ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), nhdr->n_descsz); 339 memset(build_id + nhdr->n_descsz, 0, BPF_BUILD_ID_SIZE - nhdr->n_descsz); 340 return (int) nhdr->n_descsz; 341 } 342 343 note_offs = note_offs + sizeof(Elf32_Nhdr) + 344 ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); 345 } 346 347 return -ENOENT; 348 } 349 350 /* Reads binary from *path* file and returns it in the *build_id* buffer 351 * with *size* which is expected to be at least BPF_BUILD_ID_SIZE bytes. 352 * Returns size of build id on success. On error the error value is 353 * returned. 354 */ 355 int read_build_id(const char *path, char *build_id, size_t size) 356 { 357 int fd, err = -EINVAL; 358 Elf *elf = NULL; 359 GElf_Ehdr ehdr; 360 size_t max, i; 361 362 if (size < BPF_BUILD_ID_SIZE) 363 return -EINVAL; 364 365 fd = open(path, O_RDONLY | O_CLOEXEC); 366 if (fd < 0) 367 return -errno; 368 369 (void)elf_version(EV_CURRENT); 370 371 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); 372 if (!elf) 373 goto out; 374 if (elf_kind(elf) != ELF_K_ELF) 375 goto out; 376 if (!gelf_getehdr(elf, &ehdr)) 377 goto out; 378 379 for (i = 0; i < ehdr.e_phnum; i++) { 380 GElf_Phdr mem, *phdr; 381 char *data; 382 383 phdr = gelf_getphdr(elf, i, &mem); 384 if (!phdr) 385 goto out; 386 if (phdr->p_type != PT_NOTE) 387 continue; 388 data = elf_rawfile(elf, &max); 389 if (!data) 390 goto out; 391 if (phdr->p_offset + phdr->p_memsz > max) 392 goto out; 393 err = parse_build_id_buf(data + phdr->p_offset, phdr->p_memsz, build_id); 394 if (err > 0) 395 break; 396 } 397 398 out: 399 if (elf) 400 elf_end(elf); 401 close(fd); 402 return err; 403 } 404 405 int read_trace_pipe_iter(void (*cb)(const char *str, void *data), void *data, int iter) 406 { 407 size_t buflen, n; 408 char *buf = NULL; 409 FILE *fp = NULL; 410 411 if (access(TRACEFS_PIPE, F_OK) == 0) 412 fp = fopen(TRACEFS_PIPE, "r"); 413 else 414 fp = fopen(DEBUGFS_PIPE, "r"); 415 if (!fp) 416 return -1; 417 418 /* We do not want to wait forever when iter is specified. */ 419 if (iter) 420 fcntl(fileno(fp), F_SETFL, O_NONBLOCK); 421 422 while ((n = getline(&buf, &buflen, fp) >= 0) || errno == EAGAIN) { 423 if (n > 0) 424 cb(buf, data); 425 if (iter && !(--iter)) 426 break; 427 } 428 429 free(buf); 430 if (fp) 431 fclose(fp); 432 return 0; 433 } 434 435 static void trace_pipe_cb(const char *str, void *data) 436 { 437 printf("%s", str); 438 } 439 440 void read_trace_pipe(void) 441 { 442 read_trace_pipe_iter(trace_pipe_cb, NULL, 0); 443 } 444