xref: /linux/scripts/sorttable.c (revision c0e75905caf368e19aab585d20151500e750de89)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * sorttable.c: Sort the kernel's table
4  *
5  * Added ORC unwind tables sort support and other updates:
6  * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
7  * Shile Zhang <shile.zhang@linux.alibaba.com>
8  *
9  * Copyright 2011 - 2012 Cavium, Inc.
10  *
11  * Based on code taken from recortmcount.c which is:
12  *
13  * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>.  All rights reserved.
14  *
15  * Restructured to fit Linux format, as well as other updates:
16  * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
17  */
18 
19 /*
20  * Strategy: alter the vmlinux file in-place.
21  */
22 
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #include <sys/stat.h>
26 #include <getopt.h>
27 #include <elf.h>
28 #include <fcntl.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <errno.h>
34 #include <pthread.h>
35 
36 #include <tools/be_byteshift.h>
37 #include <tools/le_byteshift.h>
38 
39 #ifndef EM_ARCOMPACT
40 #define EM_ARCOMPACT	93
41 #endif
42 
43 #ifndef EM_XTENSA
44 #define EM_XTENSA	94
45 #endif
46 
47 #ifndef EM_AARCH64
48 #define EM_AARCH64	183
49 #endif
50 
51 #ifndef EM_MICROBLAZE
52 #define EM_MICROBLAZE	189
53 #endif
54 
55 #ifndef EM_ARCV2
56 #define EM_ARCV2	195
57 #endif
58 
59 #ifndef EM_RISCV
60 #define EM_RISCV	243
61 #endif
62 
63 #ifndef EM_LOONGARCH
64 #define EM_LOONGARCH	258
65 #endif
66 
67 typedef union {
68 	Elf32_Ehdr	e32;
69 	Elf64_Ehdr	e64;
70 } Elf_Ehdr;
71 
72 typedef union {
73 	Elf32_Shdr	e32;
74 	Elf64_Shdr	e64;
75 } Elf_Shdr;
76 
77 typedef union {
78 	Elf32_Sym	e32;
79 	Elf64_Sym	e64;
80 } Elf_Sym;
81 
82 static uint32_t (*r)(const uint32_t *);
83 static uint16_t (*r2)(const uint16_t *);
84 static uint64_t (*r8)(const uint64_t *);
85 static void (*w)(uint32_t, uint32_t *);
86 typedef void (*table_sort_t)(char *, int);
87 
88 static struct elf_funcs {
89 	int (*compare_extable)(const void *a, const void *b);
90 	uint64_t (*ehdr_shoff)(Elf_Ehdr *ehdr);
91 	uint16_t (*ehdr_shstrndx)(Elf_Ehdr *ehdr);
92 	uint16_t (*ehdr_shentsize)(Elf_Ehdr *ehdr);
93 	uint16_t (*ehdr_shnum)(Elf_Ehdr *ehdr);
94 	uint64_t (*shdr_addr)(Elf_Shdr *shdr);
95 	uint64_t (*shdr_offset)(Elf_Shdr *shdr);
96 	uint64_t (*shdr_size)(Elf_Shdr *shdr);
97 	uint64_t (*shdr_entsize)(Elf_Shdr *shdr);
98 	uint32_t (*shdr_link)(Elf_Shdr *shdr);
99 	uint32_t (*shdr_name)(Elf_Shdr *shdr);
100 	uint32_t (*shdr_type)(Elf_Shdr *shdr);
101 	uint8_t (*sym_type)(Elf_Sym *sym);
102 	uint32_t (*sym_name)(Elf_Sym *sym);
103 	uint64_t (*sym_value)(Elf_Sym *sym);
104 	uint16_t (*sym_shndx)(Elf_Sym *sym);
105 } e;
106 
107 static uint64_t ehdr64_shoff(Elf_Ehdr *ehdr)
108 {
109 	return r8(&ehdr->e64.e_shoff);
110 }
111 
112 static uint64_t ehdr32_shoff(Elf_Ehdr *ehdr)
113 {
114 	return r(&ehdr->e32.e_shoff);
115 }
116 
117 static uint64_t ehdr_shoff(Elf_Ehdr *ehdr)
118 {
119 	return e.ehdr_shoff(ehdr);
120 }
121 
122 #define EHDR_HALF(fn_name)				\
123 static uint16_t ehdr64_##fn_name(Elf_Ehdr *ehdr)	\
124 {							\
125 	return r2(&ehdr->e64.e_##fn_name);		\
126 }							\
127 							\
128 static uint16_t ehdr32_##fn_name(Elf_Ehdr *ehdr)	\
129 {							\
130 	return r2(&ehdr->e32.e_##fn_name);		\
131 }							\
132 							\
133 static uint16_t ehdr_##fn_name(Elf_Ehdr *ehdr)		\
134 {							\
135 	return e.ehdr_##fn_name(ehdr);			\
136 }
137 
138 EHDR_HALF(shentsize)
139 EHDR_HALF(shstrndx)
140 EHDR_HALF(shnum)
141 
142 #define SHDR_WORD(fn_name)				\
143 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr)	\
144 {							\
145 	return r(&shdr->e64.sh_##fn_name);		\
146 }							\
147 							\
148 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr)	\
149 {							\
150 	return r(&shdr->e32.sh_##fn_name);		\
151 }							\
152 							\
153 static uint32_t shdr_##fn_name(Elf_Shdr *shdr)		\
154 {							\
155 	return e.shdr_##fn_name(shdr);			\
156 }
157 
158 #define SHDR_ADDR(fn_name)				\
159 static uint64_t shdr64_##fn_name(Elf_Shdr *shdr)	\
160 {							\
161 	return r8(&shdr->e64.sh_##fn_name);		\
162 }							\
163 							\
164 static uint64_t shdr32_##fn_name(Elf_Shdr *shdr)	\
165 {							\
166 	return r(&shdr->e32.sh_##fn_name);		\
167 }							\
168 							\
169 static uint64_t shdr_##fn_name(Elf_Shdr *shdr)		\
170 {							\
171 	return e.shdr_##fn_name(shdr);			\
172 }
173 
174 #define SHDR_WORD(fn_name)				\
175 static uint32_t shdr64_##fn_name(Elf_Shdr *shdr)	\
176 {							\
177 	return r(&shdr->e64.sh_##fn_name);		\
178 }							\
179 							\
180 static uint32_t shdr32_##fn_name(Elf_Shdr *shdr)	\
181 {							\
182 	return r(&shdr->e32.sh_##fn_name);		\
183 }							\
184 static uint32_t shdr_##fn_name(Elf_Shdr *shdr)		\
185 {							\
186 	return e.shdr_##fn_name(shdr);			\
187 }
188 
189 SHDR_ADDR(addr)
190 SHDR_ADDR(offset)
191 SHDR_ADDR(size)
192 SHDR_ADDR(entsize)
193 
194 SHDR_WORD(link)
195 SHDR_WORD(name)
196 SHDR_WORD(type)
197 
198 #define SYM_ADDR(fn_name)			\
199 static uint64_t sym64_##fn_name(Elf_Sym *sym)	\
200 {						\
201 	return r8(&sym->e64.st_##fn_name);	\
202 }						\
203 						\
204 static uint64_t sym32_##fn_name(Elf_Sym *sym)	\
205 {						\
206 	return r(&sym->e32.st_##fn_name);	\
207 }						\
208 						\
209 static uint64_t sym_##fn_name(Elf_Sym *sym)	\
210 {						\
211 	return e.sym_##fn_name(sym);		\
212 }
213 
214 #define SYM_WORD(fn_name)			\
215 static uint32_t sym64_##fn_name(Elf_Sym *sym)	\
216 {						\
217 	return r(&sym->e64.st_##fn_name);	\
218 }						\
219 						\
220 static uint32_t sym32_##fn_name(Elf_Sym *sym)	\
221 {						\
222 	return r(&sym->e32.st_##fn_name);	\
223 }						\
224 						\
225 static uint32_t sym_##fn_name(Elf_Sym *sym)	\
226 {						\
227 	return e.sym_##fn_name(sym);		\
228 }
229 
230 #define SYM_HALF(fn_name)			\
231 static uint16_t sym64_##fn_name(Elf_Sym *sym)	\
232 {						\
233 	return r2(&sym->e64.st_##fn_name);	\
234 }						\
235 						\
236 static uint16_t sym32_##fn_name(Elf_Sym *sym)	\
237 {						\
238 	return r2(&sym->e32.st_##fn_name);	\
239 }						\
240 						\
241 static uint16_t sym_##fn_name(Elf_Sym *sym)	\
242 {						\
243 	return e.sym_##fn_name(sym);		\
244 }
245 
246 static uint8_t sym64_type(Elf_Sym *sym)
247 {
248 	return ELF64_ST_TYPE(sym->e64.st_info);
249 }
250 
251 static uint8_t sym32_type(Elf_Sym *sym)
252 {
253 	return ELF32_ST_TYPE(sym->e32.st_info);
254 }
255 
256 static uint8_t sym_type(Elf_Sym *sym)
257 {
258 	return e.sym_type(sym);
259 }
260 
261 SYM_ADDR(value)
262 SYM_WORD(name)
263 SYM_HALF(shndx)
264 
265 /*
266  * Get the whole file as a programming convenience in order to avoid
267  * malloc+lseek+read+free of many pieces.  If successful, then mmap
268  * avoids copying unused pieces; else just read the whole file.
269  * Open for both read and write.
270  */
271 static void *mmap_file(char const *fname, size_t *size)
272 {
273 	int fd;
274 	struct stat sb;
275 	void *addr = NULL;
276 
277 	fd = open(fname, O_RDWR);
278 	if (fd < 0) {
279 		perror(fname);
280 		return NULL;
281 	}
282 	if (fstat(fd, &sb) < 0) {
283 		perror(fname);
284 		goto out;
285 	}
286 	if (!S_ISREG(sb.st_mode)) {
287 		fprintf(stderr, "not a regular file: %s\n", fname);
288 		goto out;
289 	}
290 
291 	addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
292 	if (addr == MAP_FAILED) {
293 		fprintf(stderr, "Could not mmap file: %s\n", fname);
294 		goto out;
295 	}
296 
297 	*size = sb.st_size;
298 
299 out:
300 	close(fd);
301 	return addr;
302 }
303 
304 static uint32_t rbe(const uint32_t *x)
305 {
306 	return get_unaligned_be32(x);
307 }
308 
309 static uint16_t r2be(const uint16_t *x)
310 {
311 	return get_unaligned_be16(x);
312 }
313 
314 static uint64_t r8be(const uint64_t *x)
315 {
316 	return get_unaligned_be64(x);
317 }
318 
319 static uint32_t rle(const uint32_t *x)
320 {
321 	return get_unaligned_le32(x);
322 }
323 
324 static uint16_t r2le(const uint16_t *x)
325 {
326 	return get_unaligned_le16(x);
327 }
328 
329 static uint64_t r8le(const uint64_t *x)
330 {
331 	return get_unaligned_le64(x);
332 }
333 
334 static void wbe(uint32_t val, uint32_t *x)
335 {
336 	put_unaligned_be32(val, x);
337 }
338 
339 static void wle(uint32_t val, uint32_t *x)
340 {
341 	put_unaligned_le32(val, x);
342 }
343 
344 /*
345  * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
346  * the way to -256..-1, to avoid conflicting with real section
347  * indices.
348  */
349 #define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1))
350 
351 static inline int is_shndx_special(unsigned int i)
352 {
353 	return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE;
354 }
355 
356 /* Accessor for sym->st_shndx, hides ugliness of "64k sections" */
357 static inline unsigned int get_secindex(unsigned int shndx,
358 					unsigned int sym_offs,
359 					const Elf32_Word *symtab_shndx_start)
360 {
361 	if (is_shndx_special(shndx))
362 		return SPECIAL(shndx);
363 	if (shndx != SHN_XINDEX)
364 		return shndx;
365 	return r(&symtab_shndx_start[sym_offs]);
366 }
367 
368 static int compare_extable_32(const void *a, const void *b)
369 {
370 	Elf32_Addr av = r(a);
371 	Elf32_Addr bv = r(b);
372 
373 	if (av < bv)
374 		return -1;
375 	return av > bv;
376 }
377 
378 static int compare_extable_64(const void *a, const void *b)
379 {
380 	Elf64_Addr av = r8(a);
381 	Elf64_Addr bv = r8(b);
382 
383 	if (av < bv)
384 		return -1;
385 	return av > bv;
386 }
387 
388 static int compare_extable(const void *a, const void *b)
389 {
390 	return e.compare_extable(a, b);
391 }
392 
393 static inline void *get_index(void *start, int entsize, int index)
394 {
395 	return start + (entsize * index);
396 }
397 
398 static int extable_ent_size;
399 static int long_size;
400 
401 
402 #ifdef UNWINDER_ORC_ENABLED
403 /* ORC unwinder only support X86_64 */
404 #include <asm/orc_types.h>
405 
406 #define ERRSTR_MAXSZ	256
407 
408 static char g_err[ERRSTR_MAXSZ];
409 static int *g_orc_ip_table;
410 static struct orc_entry *g_orc_table;
411 
412 static pthread_t orc_sort_thread;
413 
414 static inline unsigned long orc_ip(const int *ip)
415 {
416 	return (unsigned long)ip + *ip;
417 }
418 
419 static int orc_sort_cmp(const void *_a, const void *_b)
420 {
421 	struct orc_entry *orc_a, *orc_b;
422 	const int *a = g_orc_ip_table + *(int *)_a;
423 	const int *b = g_orc_ip_table + *(int *)_b;
424 	unsigned long a_val = orc_ip(a);
425 	unsigned long b_val = orc_ip(b);
426 
427 	if (a_val > b_val)
428 		return 1;
429 	if (a_val < b_val)
430 		return -1;
431 
432 	/*
433 	 * The "weak" section terminator entries need to always be on the left
434 	 * to ensure the lookup code skips them in favor of real entries.
435 	 * These terminator entries exist to handle any gaps created by
436 	 * whitelisted .o files which didn't get objtool generation.
437 	 */
438 	orc_a = g_orc_table + (a - g_orc_ip_table);
439 	orc_b = g_orc_table + (b - g_orc_ip_table);
440 	if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
441 		return 0;
442 	return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
443 }
444 
445 static void *sort_orctable(void *arg)
446 {
447 	int i;
448 	int *idxs = NULL;
449 	int *tmp_orc_ip_table = NULL;
450 	struct orc_entry *tmp_orc_table = NULL;
451 	unsigned int *orc_ip_size = (unsigned int *)arg;
452 	unsigned int num_entries = *orc_ip_size / sizeof(int);
453 	unsigned int orc_size = num_entries * sizeof(struct orc_entry);
454 
455 	idxs = (int *)malloc(*orc_ip_size);
456 	if (!idxs) {
457 		snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
458 			 strerror(errno));
459 		pthread_exit(g_err);
460 	}
461 
462 	tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
463 	if (!tmp_orc_ip_table) {
464 		snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
465 			 strerror(errno));
466 		pthread_exit(g_err);
467 	}
468 
469 	tmp_orc_table = (struct orc_entry *)malloc(orc_size);
470 	if (!tmp_orc_table) {
471 		snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
472 			 strerror(errno));
473 		pthread_exit(g_err);
474 	}
475 
476 	/* initialize indices array, convert ip_table to absolute address */
477 	for (i = 0; i < num_entries; i++) {
478 		idxs[i] = i;
479 		tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
480 	}
481 	memcpy(tmp_orc_table, g_orc_table, orc_size);
482 
483 	qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
484 
485 	for (i = 0; i < num_entries; i++) {
486 		if (idxs[i] == i)
487 			continue;
488 
489 		/* convert back to relative address */
490 		g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
491 		g_orc_table[i] = tmp_orc_table[idxs[i]];
492 	}
493 
494 	free(idxs);
495 	free(tmp_orc_ip_table);
496 	free(tmp_orc_table);
497 	pthread_exit(NULL);
498 }
499 #endif
500 
501 #ifdef MCOUNT_SORT_ENABLED
502 static pthread_t mcount_sort_thread;
503 
504 struct elf_mcount_loc {
505 	Elf_Ehdr *ehdr;
506 	Elf_Shdr *init_data_sec;
507 	uint64_t start_mcount_loc;
508 	uint64_t stop_mcount_loc;
509 };
510 
511 /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
512 static void *sort_mcount_loc(void *arg)
513 {
514 	struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
515 	uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec)
516 					+ shdr_offset(emloc->init_data_sec);
517 	uint64_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc;
518 	unsigned char *start_loc = (void *)emloc->ehdr + offset;
519 
520 	qsort(start_loc, count/long_size, long_size, compare_extable);
521 	return NULL;
522 }
523 
524 /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
525 static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec,
526 			   const char *strtab)
527 {
528 	Elf_Sym *sym, *end_sym;
529 	int symentsize = shdr_entsize(symtab_sec);
530 	int found = 0;
531 
532 	sym = (void *)emloc->ehdr + shdr_offset(symtab_sec);
533 	end_sym = (void *)sym + shdr_size(symtab_sec);
534 
535 	while (sym < end_sym) {
536 		if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) {
537 			emloc->start_mcount_loc = sym_value(sym);
538 			if (++found == 2)
539 				break;
540 		} else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) {
541 			emloc->stop_mcount_loc = sym_value(sym);
542 			if (++found == 2)
543 				break;
544 		}
545 		sym = (void *)sym + symentsize;
546 	}
547 
548 	if (!emloc->start_mcount_loc) {
549 		fprintf(stderr, "get start_mcount_loc error!");
550 		return;
551 	}
552 
553 	if (!emloc->stop_mcount_loc) {
554 		fprintf(stderr, "get stop_mcount_loc error!");
555 		return;
556 	}
557 }
558 #endif
559 
560 static int do_sort(Elf_Ehdr *ehdr,
561 		   char const *const fname,
562 		   table_sort_t custom_sort)
563 {
564 	int rc = -1;
565 	Elf_Shdr *shdr_start;
566 	Elf_Shdr *strtab_sec = NULL;
567 	Elf_Shdr *symtab_sec = NULL;
568 	Elf_Shdr *extab_sec = NULL;
569 	Elf_Shdr *string_sec;
570 	Elf_Sym *sym;
571 	const Elf_Sym *symtab;
572 	Elf32_Word *symtab_shndx = NULL;
573 	Elf_Sym *sort_needed_sym = NULL;
574 	Elf_Shdr *sort_needed_sec;
575 	uint32_t *sort_needed_loc;
576 	void *sym_start;
577 	void *sym_end;
578 	const char *secstrings;
579 	const char *strtab;
580 	char *extab_image;
581 	int sort_need_index;
582 	int symentsize;
583 	int shentsize;
584 	int idx;
585 	int i;
586 	unsigned int shnum;
587 	unsigned int shstrndx;
588 #ifdef MCOUNT_SORT_ENABLED
589 	struct elf_mcount_loc mstruct = {0};
590 #endif
591 #ifdef UNWINDER_ORC_ENABLED
592 	unsigned int orc_ip_size = 0;
593 	unsigned int orc_size = 0;
594 	unsigned int orc_num_entries = 0;
595 #endif
596 
597 	shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
598 	shentsize = ehdr_shentsize(ehdr);
599 
600 	shstrndx = ehdr_shstrndx(ehdr);
601 	if (shstrndx == SHN_XINDEX)
602 		shstrndx = shdr_link(shdr_start);
603 	string_sec = get_index(shdr_start, shentsize, shstrndx);
604 	secstrings = (const char *)ehdr + shdr_offset(string_sec);
605 
606 	shnum = ehdr_shnum(ehdr);
607 	if (shnum == SHN_UNDEF)
608 		shnum = shdr_size(shdr_start);
609 
610 	for (i = 0; i < shnum; i++) {
611 		Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
612 
613 		idx = shdr_name(shdr);
614 		if (!strcmp(secstrings + idx, "__ex_table"))
615 			extab_sec = shdr;
616 		if (!strcmp(secstrings + idx, ".symtab"))
617 			symtab_sec = shdr;
618 		if (!strcmp(secstrings + idx, ".strtab"))
619 			strtab_sec = shdr;
620 
621 		if (shdr_type(shdr) == SHT_SYMTAB_SHNDX)
622 			symtab_shndx = (Elf32_Word *)((const char *)ehdr +
623 						      shdr_offset(shdr));
624 
625 #ifdef MCOUNT_SORT_ENABLED
626 		/* locate the .init.data section in vmlinux */
627 		if (!strcmp(secstrings + idx, ".init.data"))
628 			mstruct.init_data_sec = shdr;
629 #endif
630 
631 #ifdef UNWINDER_ORC_ENABLED
632 		/* locate the ORC unwind tables */
633 		if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
634 			orc_ip_size = shdr_size(shdr);
635 			g_orc_ip_table = (int *)((void *)ehdr +
636 						   shdr_offset(shdr));
637 		}
638 		if (!strcmp(secstrings + idx, ".orc_unwind")) {
639 			orc_size = shdr_size(shdr);
640 			g_orc_table = (struct orc_entry *)((void *)ehdr +
641 							     shdr_offset(shdr));
642 		}
643 #endif
644 	} /* for loop */
645 
646 #ifdef UNWINDER_ORC_ENABLED
647 	if (!g_orc_ip_table || !g_orc_table) {
648 		fprintf(stderr,
649 			"incomplete ORC unwind tables in file: %s\n", fname);
650 		goto out;
651 	}
652 
653 	orc_num_entries = orc_ip_size / sizeof(int);
654 	if (orc_ip_size % sizeof(int) != 0 ||
655 	    orc_size % sizeof(struct orc_entry) != 0 ||
656 	    orc_num_entries != orc_size / sizeof(struct orc_entry)) {
657 		fprintf(stderr,
658 			"inconsistent ORC unwind table entries in file: %s\n",
659 			fname);
660 		goto out;
661 	}
662 
663 	/* create thread to sort ORC unwind tables concurrently */
664 	if (pthread_create(&orc_sort_thread, NULL,
665 			   sort_orctable, &orc_ip_size)) {
666 		fprintf(stderr,
667 			"pthread_create orc_sort_thread failed '%s': %s\n",
668 			strerror(errno), fname);
669 		goto out;
670 	}
671 #endif
672 	if (!extab_sec) {
673 		fprintf(stderr,	"no __ex_table in file: %s\n", fname);
674 		goto out;
675 	}
676 
677 	if (!symtab_sec) {
678 		fprintf(stderr,	"no .symtab in file: %s\n", fname);
679 		goto out;
680 	}
681 
682 	if (!strtab_sec) {
683 		fprintf(stderr,	"no .strtab in file: %s\n", fname);
684 		goto out;
685 	}
686 
687 	extab_image = (void *)ehdr + shdr_offset(extab_sec);
688 	strtab = (const char *)ehdr + shdr_offset(strtab_sec);
689 	symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec));
690 
691 #ifdef MCOUNT_SORT_ENABLED
692 	mstruct.ehdr = ehdr;
693 	get_mcount_loc(&mstruct, symtab_sec, strtab);
694 
695 	if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) {
696 		fprintf(stderr,
697 			"incomplete mcount's sort in file: %s\n",
698 			fname);
699 		goto out;
700 	}
701 
702 	/* create thread to sort mcount_loc concurrently */
703 	if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
704 		fprintf(stderr,
705 			"pthread_create mcount_sort_thread failed '%s': %s\n",
706 			strerror(errno), fname);
707 		goto out;
708 	}
709 #endif
710 
711 	if (custom_sort) {
712 		custom_sort(extab_image, shdr_size(extab_sec));
713 	} else {
714 		int num_entries = shdr_size(extab_sec) / extable_ent_size;
715 		qsort(extab_image, num_entries,
716 		      extable_ent_size, compare_extable);
717 	}
718 
719 	/* find the flag main_extable_sort_needed */
720 	sym_start = (void *)ehdr + shdr_offset(symtab_sec);
721 	sym_end = sym_start + shdr_size(symtab_sec);
722 	symentsize = shdr_entsize(symtab_sec);
723 
724 	for (sym = sym_start; (void *)sym + symentsize < sym_end;
725 	     sym = (void *)sym + symentsize) {
726 		if (sym_type(sym) != STT_OBJECT)
727 			continue;
728 		if (!strcmp(strtab + sym_name(sym),
729 			    "main_extable_sort_needed")) {
730 			sort_needed_sym = sym;
731 			break;
732 		}
733 	}
734 
735 	if (!sort_needed_sym) {
736 		fprintf(stderr,
737 			"no main_extable_sort_needed symbol in file: %s\n",
738 			fname);
739 		goto out;
740 	}
741 
742 	sort_need_index = get_secindex(sym_shndx(sym),
743 				       ((void *)sort_needed_sym - (void *)symtab) / symentsize,
744 				       symtab_shndx);
745 	sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index);
746 	sort_needed_loc = (void *)ehdr +
747 		shdr_offset(sort_needed_sec) +
748 		sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec);
749 
750 	/* extable has been sorted, clear the flag */
751 	w(0, sort_needed_loc);
752 	rc = 0;
753 
754 out:
755 #ifdef UNWINDER_ORC_ENABLED
756 	if (orc_sort_thread) {
757 		void *retval = NULL;
758 		/* wait for ORC tables sort done */
759 		rc = pthread_join(orc_sort_thread, &retval);
760 		if (rc) {
761 			fprintf(stderr,
762 				"pthread_join failed '%s': %s\n",
763 				strerror(errno), fname);
764 		} else if (retval) {
765 			rc = -1;
766 			fprintf(stderr,
767 				"failed to sort ORC tables '%s': %s\n",
768 				(char *)retval, fname);
769 		}
770 	}
771 #endif
772 
773 #ifdef MCOUNT_SORT_ENABLED
774 	if (mcount_sort_thread) {
775 		void *retval = NULL;
776 		/* wait for mcount sort done */
777 		rc = pthread_join(mcount_sort_thread, &retval);
778 		if (rc) {
779 			fprintf(stderr,
780 				"pthread_join failed '%s': %s\n",
781 				strerror(errno), fname);
782 		} else if (retval) {
783 			rc = -1;
784 			fprintf(stderr,
785 				"failed to sort mcount '%s': %s\n",
786 				(char *)retval, fname);
787 		}
788 	}
789 #endif
790 	return rc;
791 }
792 
793 static int compare_relative_table(const void *a, const void *b)
794 {
795 	int32_t av = (int32_t)r(a);
796 	int32_t bv = (int32_t)r(b);
797 
798 	if (av < bv)
799 		return -1;
800 	if (av > bv)
801 		return 1;
802 	return 0;
803 }
804 
805 static void sort_relative_table(char *extab_image, int image_size)
806 {
807 	int i = 0;
808 
809 	/*
810 	 * Do the same thing the runtime sort does, first normalize to
811 	 * being relative to the start of the section.
812 	 */
813 	while (i < image_size) {
814 		uint32_t *loc = (uint32_t *)(extab_image + i);
815 		w(r(loc) + i, loc);
816 		i += 4;
817 	}
818 
819 	qsort(extab_image, image_size / 8, 8, compare_relative_table);
820 
821 	/* Now denormalize. */
822 	i = 0;
823 	while (i < image_size) {
824 		uint32_t *loc = (uint32_t *)(extab_image + i);
825 		w(r(loc) - i, loc);
826 		i += 4;
827 	}
828 }
829 
830 static void sort_relative_table_with_data(char *extab_image, int image_size)
831 {
832 	int i = 0;
833 
834 	while (i < image_size) {
835 		uint32_t *loc = (uint32_t *)(extab_image + i);
836 
837 		w(r(loc) + i, loc);
838 		w(r(loc + 1) + i + 4, loc + 1);
839 		/* Don't touch the fixup type or data */
840 
841 		i += sizeof(uint32_t) * 3;
842 	}
843 
844 	qsort(extab_image, image_size / 12, 12, compare_relative_table);
845 
846 	i = 0;
847 	while (i < image_size) {
848 		uint32_t *loc = (uint32_t *)(extab_image + i);
849 
850 		w(r(loc) - i, loc);
851 		w(r(loc + 1) - (i + 4), loc + 1);
852 		/* Don't touch the fixup type or data */
853 
854 		i += sizeof(uint32_t) * 3;
855 	}
856 }
857 
858 static int do_file(char const *const fname, void *addr)
859 {
860 	Elf_Ehdr *ehdr = addr;
861 	table_sort_t custom_sort = NULL;
862 
863 	switch (ehdr->e32.e_ident[EI_DATA]) {
864 	case ELFDATA2LSB:
865 		r	= rle;
866 		r2	= r2le;
867 		r8	= r8le;
868 		w	= wle;
869 		break;
870 	case ELFDATA2MSB:
871 		r	= rbe;
872 		r2	= r2be;
873 		r8	= r8be;
874 		w	= wbe;
875 		break;
876 	default:
877 		fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
878 			ehdr->e32.e_ident[EI_DATA], fname);
879 		return -1;
880 	}
881 
882 	if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 ||
883 	    (r2(&ehdr->e32.e_type) != ET_EXEC && r2(&ehdr->e32.e_type) != ET_DYN) ||
884 	    ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) {
885 		fprintf(stderr, "unrecognized ET_EXEC/ET_DYN file %s\n", fname);
886 		return -1;
887 	}
888 
889 	switch (r2(&ehdr->e32.e_machine)) {
890 	case EM_386:
891 	case EM_AARCH64:
892 	case EM_LOONGARCH:
893 	case EM_RISCV:
894 	case EM_S390:
895 	case EM_X86_64:
896 		custom_sort = sort_relative_table_with_data;
897 		break;
898 	case EM_PARISC:
899 	case EM_PPC:
900 	case EM_PPC64:
901 		custom_sort = sort_relative_table;
902 		break;
903 	case EM_ARCOMPACT:
904 	case EM_ARCV2:
905 	case EM_ARM:
906 	case EM_MICROBLAZE:
907 	case EM_MIPS:
908 	case EM_XTENSA:
909 		break;
910 	default:
911 		fprintf(stderr, "unrecognized e_machine %d %s\n",
912 			r2(&ehdr->e32.e_machine), fname);
913 		return -1;
914 	}
915 
916 	switch (ehdr->e32.e_ident[EI_CLASS]) {
917 	case ELFCLASS32: {
918 		struct elf_funcs efuncs = {
919 			.compare_extable	= compare_extable_32,
920 			.ehdr_shoff		= ehdr32_shoff,
921 			.ehdr_shentsize		= ehdr32_shentsize,
922 			.ehdr_shstrndx		= ehdr32_shstrndx,
923 			.ehdr_shnum		= ehdr32_shnum,
924 			.shdr_addr		= shdr32_addr,
925 			.shdr_offset		= shdr32_offset,
926 			.shdr_link		= shdr32_link,
927 			.shdr_size		= shdr32_size,
928 			.shdr_name		= shdr32_name,
929 			.shdr_type		= shdr32_type,
930 			.shdr_entsize		= shdr32_entsize,
931 			.sym_type		= sym32_type,
932 			.sym_name		= sym32_name,
933 			.sym_value		= sym32_value,
934 			.sym_shndx		= sym32_shndx,
935 		};
936 
937 		e = efuncs;
938 		long_size		= 4;
939 		extable_ent_size	= 8;
940 
941 		if (r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) ||
942 		    r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) {
943 			fprintf(stderr,
944 				"unrecognized ET_EXEC/ET_DYN file: %s\n", fname);
945 			return -1;
946 		}
947 
948 		}
949 		break;
950 	case ELFCLASS64: {
951 		struct elf_funcs efuncs = {
952 			.compare_extable	= compare_extable_64,
953 			.ehdr_shoff		= ehdr64_shoff,
954 			.ehdr_shentsize		= ehdr64_shentsize,
955 			.ehdr_shstrndx		= ehdr64_shstrndx,
956 			.ehdr_shnum		= ehdr64_shnum,
957 			.shdr_addr		= shdr64_addr,
958 			.shdr_offset		= shdr64_offset,
959 			.shdr_link		= shdr64_link,
960 			.shdr_size		= shdr64_size,
961 			.shdr_name		= shdr64_name,
962 			.shdr_type		= shdr64_type,
963 			.shdr_entsize		= shdr64_entsize,
964 			.sym_type		= sym64_type,
965 			.sym_name		= sym64_name,
966 			.sym_value		= sym64_value,
967 			.sym_shndx		= sym64_shndx,
968 		};
969 
970 		e = efuncs;
971 		long_size		= 8;
972 		extable_ent_size	= 16;
973 
974 		if (r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) ||
975 		    r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) {
976 			fprintf(stderr,
977 				"unrecognized ET_EXEC/ET_DYN file: %s\n",
978 				fname);
979 			return -1;
980 		}
981 
982 		}
983 		break;
984 	default:
985 		fprintf(stderr, "unrecognized ELF class %d %s\n",
986 			ehdr->e32.e_ident[EI_CLASS], fname);
987 		return -1;
988 	}
989 
990 	return do_sort(ehdr, fname, custom_sort);
991 }
992 
993 int main(int argc, char *argv[])
994 {
995 	int i, n_error = 0;  /* gcc-4.3.0 false positive complaint */
996 	size_t size = 0;
997 	void *addr = NULL;
998 
999 	if (argc < 2) {
1000 		fprintf(stderr, "usage: sorttable vmlinux...\n");
1001 		return 0;
1002 	}
1003 
1004 	/* Process each file in turn, allowing deep failure. */
1005 	for (i = 1; i < argc; i++) {
1006 		addr = mmap_file(argv[i], &size);
1007 		if (!addr) {
1008 			++n_error;
1009 			continue;
1010 		}
1011 
1012 		if (do_file(argv[i], addr))
1013 			++n_error;
1014 
1015 		munmap(addr, size);
1016 	}
1017 
1018 	return !!n_error;
1019 }
1020