1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * Given several files containing CTF data, merge and uniquify that data into 30 * a single CTF section in an output file. 31 * 32 * Merges can proceed independently. As such, we perform the merges in parallel 33 * using a worker thread model. A given glob of CTF data (either all of the CTF 34 * data from a single input file, or the result of one or more merges) can only 35 * be involved in a single merge at any given time, so the process decreases in 36 * parallelism, especially towards the end, as more and more files are 37 * consolidated, finally resulting in a single merge of two large CTF graphs. 38 * Unfortunately, the last merge is also the slowest, as the two graphs being 39 * merged are each the product of merges of half of the input files. 40 * 41 * The algorithm consists of two phases, described in detail below. The first 42 * phase entails the merging of CTF data in groups of eight. The second phase 43 * takes the results of Phase I, and merges them two at a time. This disparity 44 * is due to an observation that the merge time increases at least quadratically 45 * with the size of the CTF data being merged. As such, merges of CTF graphs 46 * newly read from input files are much faster than merges of CTF graphs that 47 * are themselves the results of prior merges. 48 * 49 * A further complication is the need to ensure the repeatability of CTF merges. 50 * That is, a merge should produce the same output every time, given the same 51 * input. In both phases, this consistency requirement is met by imposing an 52 * ordering on the merge process, thus ensuring that a given set of input files 53 * are merged in the same order every time. 54 * 55 * Phase I 56 * 57 * The main thread reads the input files one by one, transforming the CTF 58 * data they contain into tdata structures. When a given file has been read 59 * and parsed, it is placed on the work queue for retrieval by worker threads. 60 * 61 * Central to Phase I is the Work In Progress (wip) array, which is used to 62 * merge batches of files in a predictable order. Files are read by the main 63 * thread, and are merged into wip array elements in round-robin order. When 64 * the number of files merged into a given array slot equals the batch size, 65 * the merged CTF graph in that array is added to the done slot in order by 66 * array slot. 67 * 68 * For example, consider a case where we have five input files, a batch size 69 * of two, a wip array size of two, and two worker threads (T1 and T2). 70 * 71 * 1. The wip array elements are assigned initial batch numbers 0 and 1. 72 * 2. T1 reads an input file from the input queue (wq_queue). This is the 73 * first input file, so it is placed into wip[0]. The second file is 74 * similarly read and placed into wip[1]. The wip array slots now contain 75 * one file each (wip_nmerged == 1). 76 * 3. T1 reads the third input file, which it merges into wip[0]. The 77 * number of files in wip[0] is equal to the batch size. 78 * 4. T2 reads the fourth input file, which it merges into wip[1]. wip[1] 79 * is now full too. 80 * 5. T2 attempts to place the contents of wip[1] on the done queue 81 * (wq_done_queue), but it can't, since the batch ID for wip[1] is 1. 82 * Batch 0 needs to be on the done queue before batch 1 can be added, so 83 * T2 blocks on wip[1]'s cv. 84 * 6. T1 attempts to place the contents of wip[0] on the done queue, and 85 * succeeds, updating wq_lastdonebatch to 0. It clears wip[0], and sets 86 * its batch ID to 2. T1 then signals wip[1]'s cv to awaken T2. 87 * 7. T2 wakes up, notices that wq_lastdonebatch is 0, which means that 88 * batch 1 can now be added. It adds wip[1] to the done queue, clears 89 * wip[1], and sets its batch ID to 3. It signals wip[0]'s cv, and 90 * restarts. 91 * 92 * The above process continues until all input files have been consumed. At 93 * this point, a pair of barriers are used to allow a single thread to move 94 * any partial batches from the wip array to the done array in batch ID order. 95 * When this is complete, wq_done_queue is moved to wq_queue, and Phase II 96 * begins. 97 * 98 * Locking Semantics (Phase I) 99 * 100 * The input queue (wq_queue) and the done queue (wq_done_queue) are 101 * protected by separate mutexes - wq_queue_lock and wq_done_queue. wip 102 * array slots are protected by their own mutexes, which must be grabbed 103 * before releasing the input queue lock. The wip array lock is dropped 104 * when the thread restarts the loop. If the array slot was full, the 105 * array lock will be held while the slot contents are added to the done 106 * queue. The done queue lock is used to protect the wip slot cv's. 107 * 108 * The pow number is protected by the queue lock. The master batch ID 109 * and last completed batch (wq_lastdonebatch) counters are protected *in 110 * Phase I* by the done queue lock. 111 * 112 * Phase II 113 * 114 * When Phase II begins, the queue consists of the merged batches from the 115 * first phase. Assume we have five batches: 116 * 117 * Q: a b c d e 118 * 119 * Using the same batch ID mechanism we used in Phase I, but without the wip 120 * array, worker threads remove two entries at a time from the beginning of 121 * the queue. These two entries are merged, and are added back to the tail 122 * of the queue, as follows: 123 * 124 * Q: a b c d e # start 125 * Q: c d e ab # a, b removed, merged, added to end 126 * Q: e ab cd # c, d removed, merged, added to end 127 * Q: cd eab # e, ab removed, merged, added to end 128 * Q: cdeab # cd, eab removed, merged, added to end 129 * 130 * When one entry remains on the queue, with no merges outstanding, Phase II 131 * finishes. We pre-determine the stopping point by pre-calculating the 132 * number of nodes that will appear on the list. In the example above, the 133 * number (wq_ninqueue) is 9. When ninqueue is 1, we conclude Phase II by 134 * signaling the main thread via wq_done_cv. 135 * 136 * Locking Semantics (Phase II) 137 * 138 * The queue (wq_queue), ninqueue, and the master batch ID and last 139 * completed batch counters are protected by wq_queue_lock. The done 140 * queue and corresponding lock are unused in Phase II as is the wip array. 141 * 142 * Uniquification 143 * 144 * We want the CTF data that goes into a given module to be as small as 145 * possible. For example, we don't want it to contain any type data that may 146 * be present in another common module. As such, after creating the master 147 * tdata_t for a given module, we can, if requested by the user, uniquify it 148 * against the tdata_t from another module (genunix in the case of the SunOS 149 * kernel). We perform a merge between the tdata_t for this module and the 150 * tdata_t from genunix. Nodes found in this module that are not present in 151 * genunix are added to a third tdata_t - the uniquified tdata_t. 152 * 153 * Additive Merges 154 * 155 * In some cases, for example if we are issuing a new version of a common 156 * module in a patch, we need to make sure that the CTF data already present 157 * in that module does not change. Changes to this data would void the CTF 158 * data in any module that uniquified against the common module. To preserve 159 * the existing data, we can perform what is known as an additive merge. In 160 * this case, a final uniquification is performed against the CTF data in the 161 * previous version of the module. The result will be the placement of new 162 * and changed data after the existing data, thus preserving the existing type 163 * ID space. 164 * 165 * Saving the result 166 * 167 * When the merges are complete, the resulting tdata_t is placed into the 168 * output file, replacing the .SUNW_ctf section (if any) already in that file. 169 * 170 * The person who changes the merging thread code in this file without updating 171 * this comment will not live to see the stock hit five. 172 */ 173 174 #include <stdio.h> 175 #include <stdlib.h> 176 #include <unistd.h> 177 #include <pthread.h> 178 #include <assert.h> 179 #if defined(sun) 180 #include <synch.h> 181 #endif 182 #include <signal.h> 183 #include <libgen.h> 184 #include <string.h> 185 #include <errno.h> 186 #if defined(sun) 187 #include <alloca.h> 188 #endif 189 #include <sys/param.h> 190 #include <sys/types.h> 191 #include <sys/mman.h> 192 #if defined(sun) 193 #include <sys/sysconf.h> 194 #endif 195 196 #include "ctf_headers.h" 197 #include "ctftools.h" 198 #include "ctfmerge.h" 199 #include "traverse.h" 200 #include "memory.h" 201 #include "fifo.h" 202 #include "barrier.h" 203 204 #pragma init(bigheap) 205 206 #define MERGE_PHASE1_BATCH_SIZE 8 207 #define MERGE_PHASE1_MAX_SLOTS 5 208 #define MERGE_INPUT_THROTTLE_LEN 10 209 210 const char *progname; 211 static char *outfile = NULL; 212 static char *tmpname = NULL; 213 static int dynsym; 214 int debug_level = DEBUG_LEVEL; 215 static size_t maxpgsize = 0x400000; 216 217 218 void 219 usage(void) 220 { 221 (void) fprintf(stderr, 222 "Usage: %s [-fgstv] -l label | -L labelenv -o outfile file ...\n" 223 " %s [-fgstv] -l label | -L labelenv -o outfile -d uniqfile\n" 224 " %*s [-g] [-D uniqlabel] file ...\n" 225 " %s [-fgstv] -l label | -L labelenv -o outfile -w withfile " 226 "file ...\n" 227 " %s [-g] -c srcfile destfile\n" 228 "\n" 229 " Note: if -L labelenv is specified and labelenv is not set in\n" 230 " the environment, a default value is used.\n", 231 progname, progname, strlen(progname), " ", 232 progname, progname); 233 } 234 235 #if defined(sun) 236 static void 237 bigheap(void) 238 { 239 size_t big, *size; 240 int sizes; 241 struct memcntl_mha mha; 242 243 /* 244 * First, get the available pagesizes. 245 */ 246 if ((sizes = getpagesizes(NULL, 0)) == -1) 247 return; 248 249 if (sizes == 1 || (size = alloca(sizeof (size_t) * sizes)) == NULL) 250 return; 251 252 if (getpagesizes(size, sizes) == -1) 253 return; 254 255 while (size[sizes - 1] > maxpgsize) 256 sizes--; 257 258 /* set big to the largest allowed page size */ 259 big = size[sizes - 1]; 260 if (big & (big - 1)) { 261 /* 262 * The largest page size is not a power of two for some 263 * inexplicable reason; return. 264 */ 265 return; 266 } 267 268 /* 269 * Now, align our break to the largest page size. 270 */ 271 if (brk((void *)((((uintptr_t)sbrk(0) - 1) & ~(big - 1)) + big)) != 0) 272 return; 273 274 /* 275 * set the preferred page size for the heap 276 */ 277 mha.mha_cmd = MHA_MAPSIZE_BSSBRK; 278 mha.mha_flags = 0; 279 mha.mha_pagesize = big; 280 281 (void) memcntl(NULL, 0, MC_HAT_ADVISE, (caddr_t)&mha, 0, 0); 282 } 283 #endif 284 285 static void 286 finalize_phase_one(workqueue_t *wq) 287 { 288 int startslot, i; 289 290 /* 291 * wip slots are cleared out only when maxbatchsz td's have been merged 292 * into them. We're not guaranteed that the number of files we're 293 * merging is a multiple of maxbatchsz, so there will be some partial 294 * groups in the wip array. Move them to the done queue in batch ID 295 * order, starting with the slot containing the next batch that would 296 * have been placed on the done queue, followed by the others. 297 * One thread will be doing this while the others wait at the barrier 298 * back in worker_thread(), so we don't need to worry about pesky things 299 * like locks. 300 */ 301 302 for (startslot = -1, i = 0; i < wq->wq_nwipslots; i++) { 303 if (wq->wq_wip[i].wip_batchid == wq->wq_lastdonebatch + 1) { 304 startslot = i; 305 break; 306 } 307 } 308 309 assert(startslot != -1); 310 311 for (i = startslot; i < startslot + wq->wq_nwipslots; i++) { 312 int slotnum = i % wq->wq_nwipslots; 313 wip_t *wipslot = &wq->wq_wip[slotnum]; 314 315 if (wipslot->wip_td != NULL) { 316 debug(2, "clearing slot %d (%d) (saving %d)\n", 317 slotnum, i, wipslot->wip_nmerged); 318 } else 319 debug(2, "clearing slot %d (%d)\n", slotnum, i); 320 321 if (wipslot->wip_td != NULL) { 322 fifo_add(wq->wq_donequeue, wipslot->wip_td); 323 wq->wq_wip[slotnum].wip_td = NULL; 324 } 325 } 326 327 wq->wq_lastdonebatch = wq->wq_next_batchid++; 328 329 debug(2, "phase one done: donequeue has %d items\n", 330 fifo_len(wq->wq_donequeue)); 331 } 332 333 static void 334 init_phase_two(workqueue_t *wq) 335 { 336 int num; 337 338 /* 339 * We're going to continually merge the first two entries on the queue, 340 * placing the result on the end, until there's nothing left to merge. 341 * At that point, everything will have been merged into one. The 342 * initial value of ninqueue needs to be equal to the total number of 343 * entries that will show up on the queue, both at the start of the 344 * phase and as generated by merges during the phase. 345 */ 346 wq->wq_ninqueue = num = fifo_len(wq->wq_donequeue); 347 while (num != 1) { 348 wq->wq_ninqueue += num / 2; 349 num = num / 2 + num % 2; 350 } 351 352 /* 353 * Move the done queue to the work queue. We won't be using the done 354 * queue in phase 2. 355 */ 356 assert(fifo_len(wq->wq_queue) == 0); 357 fifo_free(wq->wq_queue, NULL); 358 wq->wq_queue = wq->wq_donequeue; 359 } 360 361 static void 362 wip_save_work(workqueue_t *wq, wip_t *slot, int slotnum) 363 { 364 pthread_mutex_lock(&wq->wq_donequeue_lock); 365 366 while (wq->wq_lastdonebatch + 1 < slot->wip_batchid) 367 pthread_cond_wait(&slot->wip_cv, &wq->wq_donequeue_lock); 368 assert(wq->wq_lastdonebatch + 1 == slot->wip_batchid); 369 370 fifo_add(wq->wq_donequeue, slot->wip_td); 371 wq->wq_lastdonebatch++; 372 pthread_cond_signal(&wq->wq_wip[(slotnum + 1) % 373 wq->wq_nwipslots].wip_cv); 374 375 /* reset the slot for next use */ 376 slot->wip_td = NULL; 377 slot->wip_batchid = wq->wq_next_batchid++; 378 379 pthread_mutex_unlock(&wq->wq_donequeue_lock); 380 } 381 382 static void 383 wip_add_work(wip_t *slot, tdata_t *pow) 384 { 385 if (slot->wip_td == NULL) { 386 slot->wip_td = pow; 387 slot->wip_nmerged = 1; 388 } else { 389 debug(2, "%d: merging %p into %p\n", pthread_self(), 390 (void *)pow, (void *)slot->wip_td); 391 392 merge_into_master(pow, slot->wip_td, NULL, 0); 393 tdata_free(pow); 394 395 slot->wip_nmerged++; 396 } 397 } 398 399 static void 400 worker_runphase1(workqueue_t *wq) 401 { 402 wip_t *wipslot; 403 tdata_t *pow; 404 int wipslotnum, pownum; 405 406 for (;;) { 407 pthread_mutex_lock(&wq->wq_queue_lock); 408 409 while (fifo_empty(wq->wq_queue)) { 410 if (wq->wq_nomorefiles == 1) { 411 pthread_cond_broadcast(&wq->wq_work_avail); 412 pthread_mutex_unlock(&wq->wq_queue_lock); 413 414 /* on to phase 2 ... */ 415 return; 416 } 417 418 pthread_cond_wait(&wq->wq_work_avail, 419 &wq->wq_queue_lock); 420 } 421 422 /* there's work to be done! */ 423 pow = fifo_remove(wq->wq_queue); 424 pownum = wq->wq_nextpownum++; 425 pthread_cond_broadcast(&wq->wq_work_removed); 426 427 assert(pow != NULL); 428 429 /* merge it into the right slot */ 430 wipslotnum = pownum % wq->wq_nwipslots; 431 wipslot = &wq->wq_wip[wipslotnum]; 432 433 pthread_mutex_lock(&wipslot->wip_lock); 434 435 pthread_mutex_unlock(&wq->wq_queue_lock); 436 437 wip_add_work(wipslot, pow); 438 439 if (wipslot->wip_nmerged == wq->wq_maxbatchsz) 440 wip_save_work(wq, wipslot, wipslotnum); 441 442 pthread_mutex_unlock(&wipslot->wip_lock); 443 } 444 } 445 446 static void 447 worker_runphase2(workqueue_t *wq) 448 { 449 tdata_t *pow1, *pow2; 450 int batchid; 451 452 for (;;) { 453 pthread_mutex_lock(&wq->wq_queue_lock); 454 455 if (wq->wq_ninqueue == 1) { 456 pthread_cond_broadcast(&wq->wq_work_avail); 457 pthread_mutex_unlock(&wq->wq_queue_lock); 458 459 debug(2, "%d: entering p2 completion barrier\n", 460 pthread_self()); 461 if (barrier_wait(&wq->wq_bar1)) { 462 pthread_mutex_lock(&wq->wq_queue_lock); 463 wq->wq_alldone = 1; 464 pthread_cond_signal(&wq->wq_alldone_cv); 465 pthread_mutex_unlock(&wq->wq_queue_lock); 466 } 467 468 return; 469 } 470 471 if (fifo_len(wq->wq_queue) < 2) { 472 pthread_cond_wait(&wq->wq_work_avail, 473 &wq->wq_queue_lock); 474 pthread_mutex_unlock(&wq->wq_queue_lock); 475 continue; 476 } 477 478 /* there's work to be done! */ 479 pow1 = fifo_remove(wq->wq_queue); 480 pow2 = fifo_remove(wq->wq_queue); 481 wq->wq_ninqueue -= 2; 482 483 batchid = wq->wq_next_batchid++; 484 485 pthread_mutex_unlock(&wq->wq_queue_lock); 486 487 debug(2, "%d: merging %p into %p\n", pthread_self(), 488 (void *)pow1, (void *)pow2); 489 merge_into_master(pow1, pow2, NULL, 0); 490 tdata_free(pow1); 491 492 /* 493 * merging is complete. place at the tail of the queue in 494 * proper order. 495 */ 496 pthread_mutex_lock(&wq->wq_queue_lock); 497 while (wq->wq_lastdonebatch + 1 != batchid) { 498 pthread_cond_wait(&wq->wq_done_cv, 499 &wq->wq_queue_lock); 500 } 501 502 wq->wq_lastdonebatch = batchid; 503 504 fifo_add(wq->wq_queue, pow2); 505 debug(2, "%d: added %p to queue, len now %d, ninqueue %d\n", 506 pthread_self(), (void *)pow2, fifo_len(wq->wq_queue), 507 wq->wq_ninqueue); 508 pthread_cond_broadcast(&wq->wq_done_cv); 509 pthread_cond_signal(&wq->wq_work_avail); 510 pthread_mutex_unlock(&wq->wq_queue_lock); 511 } 512 } 513 514 /* 515 * Main loop for worker threads. 516 */ 517 static void 518 worker_thread(workqueue_t *wq) 519 { 520 worker_runphase1(wq); 521 522 debug(2, "%d: entering first barrier\n", pthread_self()); 523 524 if (barrier_wait(&wq->wq_bar1)) { 525 526 debug(2, "%d: doing work in first barrier\n", pthread_self()); 527 528 finalize_phase_one(wq); 529 530 init_phase_two(wq); 531 532 debug(2, "%d: ninqueue is %d, %d on queue\n", pthread_self(), 533 wq->wq_ninqueue, fifo_len(wq->wq_queue)); 534 } 535 536 debug(2, "%d: entering second barrier\n", pthread_self()); 537 538 (void) barrier_wait(&wq->wq_bar2); 539 540 debug(2, "%d: phase 1 complete\n", pthread_self()); 541 542 worker_runphase2(wq); 543 } 544 545 /* 546 * Pass a tdata_t tree, built from an input file, off to the work queue for 547 * consumption by worker threads. 548 */ 549 static int 550 merge_ctf_cb(tdata_t *td, char *name, void *arg) 551 { 552 workqueue_t *wq = arg; 553 554 debug(3, "Adding tdata %p for processing\n", (void *)td); 555 556 pthread_mutex_lock(&wq->wq_queue_lock); 557 while (fifo_len(wq->wq_queue) > wq->wq_ithrottle) { 558 debug(2, "Throttling input (len = %d, throttle = %d)\n", 559 fifo_len(wq->wq_queue), wq->wq_ithrottle); 560 pthread_cond_wait(&wq->wq_work_removed, &wq->wq_queue_lock); 561 } 562 563 fifo_add(wq->wq_queue, td); 564 debug(1, "Thread %d announcing %s\n", pthread_self(), name); 565 pthread_cond_broadcast(&wq->wq_work_avail); 566 pthread_mutex_unlock(&wq->wq_queue_lock); 567 568 return (1); 569 } 570 571 /* 572 * This program is intended to be invoked from a Makefile, as part of the build. 573 * As such, in the event of a failure or user-initiated interrupt (^C), we need 574 * to ensure that a subsequent re-make will cause ctfmerge to be executed again. 575 * Unfortunately, ctfmerge will usually be invoked directly after (and as part 576 * of the same Makefile rule as) a link, and will operate on the linked file 577 * in place. If we merely exit upon receipt of a SIGINT, a subsequent make 578 * will notice that the *linked* file is newer than the object files, and thus 579 * will not reinvoke ctfmerge. The only way to ensure that a subsequent make 580 * reinvokes ctfmerge, is to remove the file to which we are adding CTF 581 * data (confusingly named the output file). This means that the link will need 582 * to happen again, but links are generally fast, and we can't allow the merge 583 * to be skipped. 584 * 585 * Another possibility would be to block SIGINT entirely - to always run to 586 * completion. The run time of ctfmerge can, however, be measured in minutes 587 * in some cases, so this is not a valid option. 588 */ 589 static void 590 handle_sig(int sig) 591 { 592 terminate("Caught signal %d - exiting\n", sig); 593 } 594 595 static void 596 terminate_cleanup(void) 597 { 598 int dounlink = getenv("CTFMERGE_TERMINATE_NO_UNLINK") ? 0 : 1; 599 600 if (tmpname != NULL && dounlink) 601 unlink(tmpname); 602 603 if (outfile == NULL) 604 return; 605 606 #if !defined(__FreeBSD__) 607 if (dounlink) { 608 fprintf(stderr, "Removing %s\n", outfile); 609 unlink(outfile); 610 } 611 #endif 612 } 613 614 static void 615 copy_ctf_data(char *srcfile, char *destfile, int keep_stabs) 616 { 617 tdata_t *srctd; 618 619 if (read_ctf(&srcfile, 1, NULL, read_ctf_save_cb, &srctd, 1) == 0) 620 terminate("No CTF data found in source file %s\n", srcfile); 621 622 tmpname = mktmpname(destfile, ".ctf"); 623 write_ctf(srctd, destfile, tmpname, CTF_COMPRESS | keep_stabs); 624 if (rename(tmpname, destfile) != 0) { 625 terminate("Couldn't rename temp file %s to %s", tmpname, 626 destfile); 627 } 628 free(tmpname); 629 tdata_free(srctd); 630 } 631 632 static void 633 wq_init(workqueue_t *wq, int nfiles) 634 { 635 int throttle, nslots, i; 636 637 if (getenv("CTFMERGE_MAX_SLOTS")) 638 nslots = atoi(getenv("CTFMERGE_MAX_SLOTS")); 639 else 640 nslots = MERGE_PHASE1_MAX_SLOTS; 641 642 if (getenv("CTFMERGE_PHASE1_BATCH_SIZE")) 643 wq->wq_maxbatchsz = atoi(getenv("CTFMERGE_PHASE1_BATCH_SIZE")); 644 else 645 wq->wq_maxbatchsz = MERGE_PHASE1_BATCH_SIZE; 646 647 nslots = MIN(nslots, (nfiles + wq->wq_maxbatchsz - 1) / 648 wq->wq_maxbatchsz); 649 650 wq->wq_wip = xcalloc(sizeof (wip_t) * nslots); 651 wq->wq_nwipslots = nslots; 652 wq->wq_nthreads = MIN(sysconf(_SC_NPROCESSORS_ONLN) * 3 / 2, nslots); 653 654 if (getenv("CTFMERGE_INPUT_THROTTLE")) 655 throttle = atoi(getenv("CTFMERGE_INPUT_THROTTLE")); 656 else 657 throttle = MERGE_INPUT_THROTTLE_LEN; 658 wq->wq_ithrottle = throttle * wq->wq_nthreads; 659 660 debug(1, "Using %d slots, %d threads\n", wq->wq_nwipslots, 661 wq->wq_nthreads); 662 663 wq->wq_next_batchid = 0; 664 665 for (i = 0; i < nslots; i++) { 666 pthread_mutex_init(&wq->wq_wip[i].wip_lock, NULL); 667 wq->wq_wip[i].wip_batchid = wq->wq_next_batchid++; 668 } 669 670 pthread_mutex_init(&wq->wq_queue_lock, NULL); 671 wq->wq_queue = fifo_new(); 672 pthread_cond_init(&wq->wq_work_avail, NULL); 673 pthread_cond_init(&wq->wq_work_removed, NULL); 674 wq->wq_ninqueue = nfiles; 675 wq->wq_nextpownum = 0; 676 677 pthread_mutex_init(&wq->wq_donequeue_lock, NULL); 678 wq->wq_donequeue = fifo_new(); 679 wq->wq_lastdonebatch = -1; 680 681 pthread_cond_init(&wq->wq_done_cv, NULL); 682 683 pthread_cond_init(&wq->wq_alldone_cv, NULL); 684 wq->wq_alldone = 0; 685 686 barrier_init(&wq->wq_bar1, wq->wq_nthreads); 687 barrier_init(&wq->wq_bar2, wq->wq_nthreads); 688 689 wq->wq_nomorefiles = 0; 690 } 691 692 static void 693 start_threads(workqueue_t *wq) 694 { 695 pthread_t thrid; 696 sigset_t sets; 697 int i; 698 699 sigemptyset(&sets); 700 sigaddset(&sets, SIGINT); 701 sigaddset(&sets, SIGQUIT); 702 sigaddset(&sets, SIGTERM); 703 pthread_sigmask(SIG_BLOCK, &sets, NULL); 704 705 for (i = 0; i < wq->wq_nthreads; i++) { 706 pthread_create(&thrid, NULL, (void *(*)(void *))worker_thread, 707 wq); 708 } 709 710 #if defined(sun) 711 sigset(SIGINT, handle_sig); 712 sigset(SIGQUIT, handle_sig); 713 sigset(SIGTERM, handle_sig); 714 #else 715 signal(SIGINT, handle_sig); 716 signal(SIGQUIT, handle_sig); 717 signal(SIGTERM, handle_sig); 718 #endif 719 pthread_sigmask(SIG_UNBLOCK, &sets, NULL); 720 } 721 722 static int 723 strcompare(const void *p1, const void *p2) 724 { 725 char *s1 = *((char **)p1); 726 char *s2 = *((char **)p2); 727 728 return (strcmp(s1, s2)); 729 } 730 731 int 732 main(int argc, char **argv) 733 { 734 workqueue_t wq; 735 tdata_t *mstrtd, *savetd; 736 char *uniqfile = NULL, *uniqlabel = NULL; 737 char *withfile = NULL; 738 char *label = NULL; 739 char **ifiles, **tifiles; 740 int verbose = 0, docopy = 0; 741 int write_fuzzy_match = 0; 742 int keep_stabs = 0; 743 int require_ctf = 0; 744 int nifiles, nielems; 745 int c, i, idx, tidx, err; 746 747 progname = basename(argv[0]); 748 749 if (getenv("CTFMERGE_DEBUG_LEVEL")) 750 debug_level = atoi(getenv("CTFMERGE_DEBUG_LEVEL")); 751 752 err = 0; 753 while ((c = getopt(argc, argv, ":cd:D:fgl:L:o:tvw:s")) != EOF) { 754 switch (c) { 755 case 'c': 756 docopy = 1; 757 break; 758 case 'd': 759 /* Uniquify against `uniqfile' */ 760 uniqfile = optarg; 761 break; 762 case 'D': 763 /* Uniquify against label `uniqlabel' in `uniqfile' */ 764 uniqlabel = optarg; 765 break; 766 case 'f': 767 write_fuzzy_match = CTF_FUZZY_MATCH; 768 break; 769 case 'g': 770 keep_stabs = CTF_KEEP_STABS; 771 break; 772 case 'l': 773 /* Label merged types with `label' */ 774 label = optarg; 775 break; 776 case 'L': 777 /* Label merged types with getenv(`label`) */ 778 if ((label = getenv(optarg)) == NULL) 779 label = CTF_DEFAULT_LABEL; 780 break; 781 case 'o': 782 /* Place merged types in CTF section in `outfile' */ 783 outfile = optarg; 784 break; 785 case 't': 786 /* Insist *all* object files built from C have CTF */ 787 require_ctf = 1; 788 break; 789 case 'v': 790 /* More debugging information */ 791 verbose = 1; 792 break; 793 case 'w': 794 /* Additive merge with data from `withfile' */ 795 withfile = optarg; 796 break; 797 case 's': 798 /* use the dynsym rather than the symtab */ 799 dynsym = CTF_USE_DYNSYM; 800 break; 801 default: 802 usage(); 803 exit(2); 804 } 805 } 806 807 /* Validate arguments */ 808 if (docopy) { 809 if (uniqfile != NULL || uniqlabel != NULL || label != NULL || 810 outfile != NULL || withfile != NULL || dynsym != 0) 811 err++; 812 813 if (argc - optind != 2) 814 err++; 815 } else { 816 if (uniqfile != NULL && withfile != NULL) 817 err++; 818 819 if (uniqlabel != NULL && uniqfile == NULL) 820 err++; 821 822 if (outfile == NULL || label == NULL) 823 err++; 824 825 if (argc - optind == 0) 826 err++; 827 } 828 829 if (err) { 830 usage(); 831 exit(2); 832 } 833 834 if (getenv("STRIPSTABS_KEEP_STABS") != NULL) 835 keep_stabs = CTF_KEEP_STABS; 836 837 if (uniqfile && access(uniqfile, R_OK) != 0) { 838 warning("Uniquification file %s couldn't be opened and " 839 "will be ignored.\n", uniqfile); 840 uniqfile = NULL; 841 } 842 if (withfile && access(withfile, R_OK) != 0) { 843 warning("With file %s couldn't be opened and will be " 844 "ignored.\n", withfile); 845 withfile = NULL; 846 } 847 if (outfile && access(outfile, R_OK|W_OK) != 0) 848 terminate("Cannot open output file %s for r/w", outfile); 849 850 /* 851 * This is ugly, but we don't want to have to have a separate tool 852 * (yet) just for copying an ELF section with our specific requirements, 853 * so we shoe-horn a copier into ctfmerge. 854 */ 855 if (docopy) { 856 copy_ctf_data(argv[optind], argv[optind + 1], keep_stabs); 857 858 exit(0); 859 } 860 861 set_terminate_cleanup(terminate_cleanup); 862 863 /* Sort the input files and strip out duplicates */ 864 nifiles = argc - optind; 865 ifiles = xmalloc(sizeof (char *) * nifiles); 866 tifiles = xmalloc(sizeof (char *) * nifiles); 867 868 for (i = 0; i < nifiles; i++) 869 tifiles[i] = argv[optind + i]; 870 qsort(tifiles, nifiles, sizeof (char *), (int (*)())strcompare); 871 872 ifiles[0] = tifiles[0]; 873 for (idx = 0, tidx = 1; tidx < nifiles; tidx++) { 874 if (strcmp(ifiles[idx], tifiles[tidx]) != 0) 875 ifiles[++idx] = tifiles[tidx]; 876 } 877 nifiles = idx + 1; 878 879 /* Make sure they all exist */ 880 if ((nielems = count_files(ifiles, nifiles)) < 0) 881 terminate("Some input files were inaccessible\n"); 882 883 /* Prepare for the merge */ 884 wq_init(&wq, nielems); 885 886 start_threads(&wq); 887 888 /* 889 * Start the merge 890 * 891 * We're reading everything from each of the object files, so we 892 * don't need to specify labels. 893 */ 894 if (read_ctf(ifiles, nifiles, NULL, merge_ctf_cb, 895 &wq, require_ctf) == 0) { 896 /* 897 * If we're verifying that C files have CTF, it's safe to 898 * assume that in this case, we're building only from assembly 899 * inputs. 900 */ 901 if (require_ctf) 902 exit(0); 903 terminate("No ctf sections found to merge\n"); 904 } 905 906 pthread_mutex_lock(&wq.wq_queue_lock); 907 wq.wq_nomorefiles = 1; 908 pthread_cond_broadcast(&wq.wq_work_avail); 909 pthread_mutex_unlock(&wq.wq_queue_lock); 910 911 pthread_mutex_lock(&wq.wq_queue_lock); 912 while (wq.wq_alldone == 0) 913 pthread_cond_wait(&wq.wq_alldone_cv, &wq.wq_queue_lock); 914 pthread_mutex_unlock(&wq.wq_queue_lock); 915 916 /* 917 * All requested files have been merged, with the resulting tree in 918 * mstrtd. savetd is the tree that will be placed into the output file. 919 * 920 * Regardless of whether we're doing a normal uniquification or an 921 * additive merge, we need a type tree that has been uniquified 922 * against uniqfile or withfile, as appropriate. 923 * 924 * If we're doing a uniquification, we stuff the resulting tree into 925 * outfile. Otherwise, we add the tree to the tree already in withfile. 926 */ 927 assert(fifo_len(wq.wq_queue) == 1); 928 mstrtd = fifo_remove(wq.wq_queue); 929 930 if (verbose || debug_level) { 931 debug(2, "Statistics for td %p\n", (void *)mstrtd); 932 933 iidesc_stats(mstrtd->td_iihash); 934 } 935 936 if (uniqfile != NULL || withfile != NULL) { 937 char *reffile, *reflabel = NULL; 938 tdata_t *reftd; 939 940 if (uniqfile != NULL) { 941 reffile = uniqfile; 942 reflabel = uniqlabel; 943 } else 944 reffile = withfile; 945 946 if (read_ctf(&reffile, 1, reflabel, read_ctf_save_cb, 947 &reftd, require_ctf) == 0) { 948 terminate("No CTF data found in reference file %s\n", 949 reffile); 950 } 951 952 savetd = tdata_new(); 953 954 if (CTF_TYPE_ISCHILD(reftd->td_nextid)) 955 terminate("No room for additional types in master\n"); 956 957 savetd->td_nextid = withfile ? reftd->td_nextid : 958 CTF_INDEX_TO_TYPE(1, TRUE); 959 merge_into_master(mstrtd, reftd, savetd, 0); 960 961 tdata_label_add(savetd, label, CTF_LABEL_LASTIDX); 962 963 if (withfile) { 964 /* 965 * savetd holds the new data to be added to the withfile 966 */ 967 tdata_t *withtd = reftd; 968 969 tdata_merge(withtd, savetd); 970 971 savetd = withtd; 972 } else { 973 char uniqname[MAXPATHLEN]; 974 labelent_t *parle; 975 976 parle = tdata_label_top(reftd); 977 978 savetd->td_parlabel = xstrdup(parle->le_name); 979 980 strncpy(uniqname, reffile, sizeof (uniqname)); 981 uniqname[MAXPATHLEN - 1] = '\0'; 982 savetd->td_parname = xstrdup(basename(uniqname)); 983 } 984 985 } else { 986 /* 987 * No post processing. Write the merged tree as-is into the 988 * output file. 989 */ 990 tdata_label_free(mstrtd); 991 tdata_label_add(mstrtd, label, CTF_LABEL_LASTIDX); 992 993 savetd = mstrtd; 994 } 995 996 tmpname = mktmpname(outfile, ".ctf"); 997 write_ctf(savetd, outfile, tmpname, 998 CTF_COMPRESS | write_fuzzy_match | dynsym | keep_stabs); 999 if (rename(tmpname, outfile) != 0) 1000 terminate("Couldn't rename output temp file %s", tmpname); 1001 free(tmpname); 1002 1003 return (0); 1004 } 1005