1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * sorttable.h 4 * 5 * Added ORC unwind tables sort support and other updates: 6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: 7 * Shile Zhang <shile.zhang@linux.alibaba.com> 8 * 9 * Copyright 2011 - 2012 Cavium, Inc. 10 * 11 * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: 12 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> 13 * 14 * Some of this code was taken out of recordmcount.h written by: 15 * 16 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. 17 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. 18 */ 19 20 #undef extable_ent_size 21 #undef compare_extable 22 #undef get_mcount_loc 23 #undef sort_mcount_loc 24 #undef elf_mcount_loc 25 #undef do_sort 26 #undef Elf_Addr 27 #undef Elf_Ehdr 28 #undef Elf_Shdr 29 #undef Elf_Rel 30 #undef Elf_Rela 31 #undef Elf_Sym 32 #undef ELF_R_SYM 33 #undef Elf_r_sym 34 #undef ELF_R_INFO 35 #undef Elf_r_info 36 #undef ELF_ST_BIND 37 #undef ELF_ST_TYPE 38 #undef fn_ELF_R_SYM 39 #undef fn_ELF_R_INFO 40 #undef uint_t 41 #undef _r 42 #undef _w 43 44 #ifdef SORTTABLE_64 45 # define extable_ent_size 16 46 # define compare_extable compare_extable_64 47 # define get_mcount_loc get_mcount_loc_64 48 # define sort_mcount_loc sort_mcount_loc_64 49 # define elf_mcount_loc elf_mcount_loc_64 50 # define do_sort do_sort_64 51 # define Elf_Addr Elf64_Addr 52 # define Elf_Ehdr Elf64_Ehdr 53 # define Elf_Shdr Elf64_Shdr 54 # define Elf_Rel Elf64_Rel 55 # define Elf_Rela Elf64_Rela 56 # define Elf_Sym Elf64_Sym 57 # define ELF_R_SYM ELF64_R_SYM 58 # define Elf_r_sym Elf64_r_sym 59 # define ELF_R_INFO ELF64_R_INFO 60 # define Elf_r_info Elf64_r_info 61 # define ELF_ST_BIND ELF64_ST_BIND 62 # define ELF_ST_TYPE ELF64_ST_TYPE 63 # define fn_ELF_R_SYM fn_ELF64_R_SYM 64 # define fn_ELF_R_INFO fn_ELF64_R_INFO 65 # define uint_t uint64_t 66 # define _r r8 67 # define _w w8 68 #else 69 # define extable_ent_size 8 70 # define compare_extable compare_extable_32 71 # define get_mcount_loc get_mcount_loc_32 72 # define sort_mcount_loc sort_mcount_loc_32 73 # define elf_mcount_loc elf_mcount_loc_32 74 # define do_sort do_sort_32 75 # define Elf_Addr Elf32_Addr 76 # define Elf_Ehdr Elf32_Ehdr 77 # define Elf_Shdr Elf32_Shdr 78 # define Elf_Rel Elf32_Rel 79 # define Elf_Rela Elf32_Rela 80 # define Elf_Sym Elf32_Sym 81 # define ELF_R_SYM ELF32_R_SYM 82 # define Elf_r_sym Elf32_r_sym 83 # define ELF_R_INFO ELF32_R_INFO 84 # define Elf_r_info Elf32_r_info 85 # define ELF_ST_BIND ELF32_ST_BIND 86 # define ELF_ST_TYPE ELF32_ST_TYPE 87 # define fn_ELF_R_SYM fn_ELF32_R_SYM 88 # define fn_ELF_R_INFO fn_ELF32_R_INFO 89 # define uint_t uint32_t 90 # define _r r 91 # define _w w 92 #endif 93 94 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 95 /* ORC unwinder only support X86_64 */ 96 #include <asm/orc_types.h> 97 98 #define ERRSTR_MAXSZ 256 99 100 char g_err[ERRSTR_MAXSZ]; 101 int *g_orc_ip_table; 102 struct orc_entry *g_orc_table; 103 104 pthread_t orc_sort_thread; 105 106 static inline unsigned long orc_ip(const int *ip) 107 { 108 return (unsigned long)ip + *ip; 109 } 110 111 static int orc_sort_cmp(const void *_a, const void *_b) 112 { 113 struct orc_entry *orc_a, *orc_b; 114 const int *a = g_orc_ip_table + *(int *)_a; 115 const int *b = g_orc_ip_table + *(int *)_b; 116 unsigned long a_val = orc_ip(a); 117 unsigned long b_val = orc_ip(b); 118 119 if (a_val > b_val) 120 return 1; 121 if (a_val < b_val) 122 return -1; 123 124 /* 125 * The "weak" section terminator entries need to always be on the left 126 * to ensure the lookup code skips them in favor of real entries. 127 * These terminator entries exist to handle any gaps created by 128 * whitelisted .o files which didn't get objtool generation. 129 */ 130 orc_a = g_orc_table + (a - g_orc_ip_table); 131 orc_b = g_orc_table + (b - g_orc_ip_table); 132 if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED) 133 return 0; 134 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 135 } 136 137 static void *sort_orctable(void *arg) 138 { 139 int i; 140 int *idxs = NULL; 141 int *tmp_orc_ip_table = NULL; 142 struct orc_entry *tmp_orc_table = NULL; 143 unsigned int *orc_ip_size = (unsigned int *)arg; 144 unsigned int num_entries = *orc_ip_size / sizeof(int); 145 unsigned int orc_size = num_entries * sizeof(struct orc_entry); 146 147 idxs = (int *)malloc(*orc_ip_size); 148 if (!idxs) { 149 snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", 150 strerror(errno)); 151 pthread_exit(g_err); 152 } 153 154 tmp_orc_ip_table = (int *)malloc(*orc_ip_size); 155 if (!tmp_orc_ip_table) { 156 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", 157 strerror(errno)); 158 pthread_exit(g_err); 159 } 160 161 tmp_orc_table = (struct orc_entry *)malloc(orc_size); 162 if (!tmp_orc_table) { 163 snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", 164 strerror(errno)); 165 pthread_exit(g_err); 166 } 167 168 /* initialize indices array, convert ip_table to absolute address */ 169 for (i = 0; i < num_entries; i++) { 170 idxs[i] = i; 171 tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); 172 } 173 memcpy(tmp_orc_table, g_orc_table, orc_size); 174 175 qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); 176 177 for (i = 0; i < num_entries; i++) { 178 if (idxs[i] == i) 179 continue; 180 181 /* convert back to relative address */ 182 g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); 183 g_orc_table[i] = tmp_orc_table[idxs[i]]; 184 } 185 186 free(idxs); 187 free(tmp_orc_ip_table); 188 free(tmp_orc_table); 189 pthread_exit(NULL); 190 } 191 #endif 192 193 static int compare_extable(const void *a, const void *b) 194 { 195 Elf_Addr av = _r(a); 196 Elf_Addr bv = _r(b); 197 198 if (av < bv) 199 return -1; 200 if (av > bv) 201 return 1; 202 return 0; 203 } 204 #ifdef MCOUNT_SORT_ENABLED 205 pthread_t mcount_sort_thread; 206 207 struct elf_mcount_loc { 208 Elf_Ehdr *ehdr; 209 Elf_Shdr *init_data_sec; 210 uint_t start_mcount_loc; 211 uint_t stop_mcount_loc; 212 }; 213 214 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ 215 static void *sort_mcount_loc(void *arg) 216 { 217 struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; 218 uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) 219 + _r(&(emloc->init_data_sec)->sh_offset); 220 uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; 221 unsigned char *start_loc = (void *)emloc->ehdr + offset; 222 223 qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); 224 return NULL; 225 } 226 227 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ 228 static void get_mcount_loc(uint_t *_start, uint_t *_stop) 229 { 230 FILE *file_start, *file_stop; 231 char start_buff[20]; 232 char stop_buff[20]; 233 int len = 0; 234 235 file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); 236 if (!file_start) { 237 fprintf(stderr, "get start_mcount_loc error!"); 238 return; 239 } 240 241 file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); 242 if (!file_stop) { 243 fprintf(stderr, "get stop_mcount_loc error!"); 244 pclose(file_start); 245 return; 246 } 247 248 while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { 249 len = strlen(start_buff); 250 start_buff[len - 1] = '\0'; 251 } 252 *_start = strtoul(start_buff, NULL, 16); 253 254 while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { 255 len = strlen(stop_buff); 256 stop_buff[len - 1] = '\0'; 257 } 258 *_stop = strtoul(stop_buff, NULL, 16); 259 260 pclose(file_start); 261 pclose(file_stop); 262 } 263 #endif 264 static int do_sort(Elf_Ehdr *ehdr, 265 char const *const fname, 266 table_sort_t custom_sort) 267 { 268 int rc = -1; 269 Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); 270 Elf_Shdr *strtab_sec = NULL; 271 Elf_Shdr *symtab_sec = NULL; 272 Elf_Shdr *extab_sec = NULL; 273 Elf_Sym *sym; 274 const Elf_Sym *symtab; 275 Elf32_Word *symtab_shndx = NULL; 276 Elf_Sym *sort_needed_sym = NULL; 277 Elf_Shdr *sort_needed_sec; 278 Elf_Rel *relocs = NULL; 279 int relocs_size = 0; 280 uint32_t *sort_needed_loc; 281 const char *secstrings; 282 const char *strtab; 283 char *extab_image; 284 int extab_index = 0; 285 int i; 286 int idx; 287 unsigned int shnum; 288 unsigned int shstrndx; 289 #ifdef MCOUNT_SORT_ENABLED 290 struct elf_mcount_loc mstruct = {0}; 291 uint_t _start_mcount_loc = 0; 292 uint_t _stop_mcount_loc = 0; 293 #endif 294 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 295 unsigned int orc_ip_size = 0; 296 unsigned int orc_size = 0; 297 unsigned int orc_num_entries = 0; 298 #endif 299 300 shstrndx = r2(&ehdr->e_shstrndx); 301 if (shstrndx == SHN_XINDEX) 302 shstrndx = r(&shdr[0].sh_link); 303 secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); 304 305 shnum = r2(&ehdr->e_shnum); 306 if (shnum == SHN_UNDEF) 307 shnum = _r(&shdr[0].sh_size); 308 309 for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { 310 idx = r(&s->sh_name); 311 if (!strcmp(secstrings + idx, "__ex_table")) { 312 extab_sec = s; 313 extab_index = i; 314 } 315 if (!strcmp(secstrings + idx, ".symtab")) 316 symtab_sec = s; 317 if (!strcmp(secstrings + idx, ".strtab")) 318 strtab_sec = s; 319 320 if ((r(&s->sh_type) == SHT_REL || 321 r(&s->sh_type) == SHT_RELA) && 322 r(&s->sh_info) == extab_index) { 323 relocs = (void *)ehdr + _r(&s->sh_offset); 324 relocs_size = _r(&s->sh_size); 325 } 326 if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) 327 symtab_shndx = (Elf32_Word *)((const char *)ehdr + 328 _r(&s->sh_offset)); 329 330 #ifdef MCOUNT_SORT_ENABLED 331 /* locate the .init.data section in vmlinux */ 332 if (!strcmp(secstrings + idx, ".init.data")) { 333 get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); 334 mstruct.ehdr = ehdr; 335 mstruct.init_data_sec = s; 336 mstruct.start_mcount_loc = _start_mcount_loc; 337 mstruct.stop_mcount_loc = _stop_mcount_loc; 338 } 339 #endif 340 341 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 342 /* locate the ORC unwind tables */ 343 if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { 344 orc_ip_size = s->sh_size; 345 g_orc_ip_table = (int *)((void *)ehdr + 346 s->sh_offset); 347 } 348 if (!strcmp(secstrings + idx, ".orc_unwind")) { 349 orc_size = s->sh_size; 350 g_orc_table = (struct orc_entry *)((void *)ehdr + 351 s->sh_offset); 352 } 353 #endif 354 } /* for loop */ 355 356 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 357 if (!g_orc_ip_table || !g_orc_table) { 358 fprintf(stderr, 359 "incomplete ORC unwind tables in file: %s\n", fname); 360 goto out; 361 } 362 363 orc_num_entries = orc_ip_size / sizeof(int); 364 if (orc_ip_size % sizeof(int) != 0 || 365 orc_size % sizeof(struct orc_entry) != 0 || 366 orc_num_entries != orc_size / sizeof(struct orc_entry)) { 367 fprintf(stderr, 368 "inconsistent ORC unwind table entries in file: %s\n", 369 fname); 370 goto out; 371 } 372 373 /* create thread to sort ORC unwind tables concurrently */ 374 if (pthread_create(&orc_sort_thread, NULL, 375 sort_orctable, &orc_ip_size)) { 376 fprintf(stderr, 377 "pthread_create orc_sort_thread failed '%s': %s\n", 378 strerror(errno), fname); 379 goto out; 380 } 381 #endif 382 383 #ifdef MCOUNT_SORT_ENABLED 384 if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { 385 fprintf(stderr, 386 "incomplete mcount's sort in file: %s\n", 387 fname); 388 goto out; 389 } 390 391 /* create thread to sort mcount_loc concurrently */ 392 if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { 393 fprintf(stderr, 394 "pthread_create mcount_sort_thread failed '%s': %s\n", 395 strerror(errno), fname); 396 goto out; 397 } 398 #endif 399 if (!extab_sec) { 400 fprintf(stderr, "no __ex_table in file: %s\n", fname); 401 goto out; 402 } 403 404 if (!symtab_sec) { 405 fprintf(stderr, "no .symtab in file: %s\n", fname); 406 goto out; 407 } 408 409 if (!strtab_sec) { 410 fprintf(stderr, "no .strtab in file: %s\n", fname); 411 goto out; 412 } 413 414 extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); 415 strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); 416 symtab = (const Elf_Sym *)((const char *)ehdr + 417 _r(&symtab_sec->sh_offset)); 418 419 if (custom_sort) { 420 custom_sort(extab_image, _r(&extab_sec->sh_size)); 421 } else { 422 int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; 423 qsort(extab_image, num_entries, 424 extable_ent_size, compare_extable); 425 } 426 427 /* If there were relocations, we no longer need them. */ 428 if (relocs) 429 memset(relocs, 0, relocs_size); 430 431 /* find the flag main_extable_sort_needed */ 432 for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); 433 sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); 434 sym++) { 435 if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) 436 continue; 437 if (!strcmp(strtab + r(&sym->st_name), 438 "main_extable_sort_needed")) { 439 sort_needed_sym = sym; 440 break; 441 } 442 } 443 444 if (!sort_needed_sym) { 445 fprintf(stderr, 446 "no main_extable_sort_needed symbol in file: %s\n", 447 fname); 448 goto out; 449 } 450 451 sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), 452 sort_needed_sym - symtab, 453 symtab_shndx)]; 454 sort_needed_loc = (void *)ehdr + 455 _r(&sort_needed_sec->sh_offset) + 456 _r(&sort_needed_sym->st_value) - 457 _r(&sort_needed_sec->sh_addr); 458 459 /* extable has been sorted, clear the flag */ 460 w(0, sort_needed_loc); 461 rc = 0; 462 463 out: 464 #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) 465 if (orc_sort_thread) { 466 void *retval = NULL; 467 /* wait for ORC tables sort done */ 468 rc = pthread_join(orc_sort_thread, &retval); 469 if (rc) { 470 fprintf(stderr, 471 "pthread_join failed '%s': %s\n", 472 strerror(errno), fname); 473 } else if (retval) { 474 rc = -1; 475 fprintf(stderr, 476 "failed to sort ORC tables '%s': %s\n", 477 (char *)retval, fname); 478 } 479 } 480 #endif 481 482 #ifdef MCOUNT_SORT_ENABLED 483 if (mcount_sort_thread) { 484 void *retval = NULL; 485 /* wait for mcount sort done */ 486 rc = pthread_join(mcount_sort_thread, &retval); 487 if (rc) { 488 fprintf(stderr, 489 "pthread_join failed '%s': %s\n", 490 strerror(errno), fname); 491 } else if (retval) { 492 rc = -1; 493 fprintf(stderr, 494 "failed to sort mcount '%s': %s\n", 495 (char *)retval, fname); 496 } 497 } 498 #endif 499 return rc; 500 } 501