1#!@TOOLS_PYTHON@ -Es 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23# Copyright 2021 OmniOS Community Edition (OmniOSce) Association. 24# 25 26# 27# wsdiff(1) is a tool that can be used to determine which compiled objects 28# have changed as a result of a given source change. Developers backporting 29# new features, RFEs and bug fixes need to be able to identify the set of 30# patch deliverables necessary for feature/fix realization on a patched system. 31# 32# The tool works by comparing objects in two trees/proto areas (one build with, 33# and without the source changes. 34# 35# Using wsdiff(1) is fairly simple: 36# - Bringover to a fresh workspace 37# - Perform a full non-debug build (clobber if workspace isn't fresh) 38# - Move the proto area aside, call it proto.old, or something. 39# - Integrate your changes to the workspace 40# - Perform another full non-debug clobber build. 41# - Use wsdiff(1) to see what changed: 42# $ wsdiff proto.old proto 43# 44# By default, wsdiff will print the list of changed objects / deliverables to 45# stdout. If a results file is specified via -r, the list of differing objects, 46# and details about why wsdiff(1) thinks they are different will be logged to 47# the results file. 48# 49# By invoking nightly(1) with the -w option to NIGHTLY_FLAGS, nightly(1) will 50# use wsdiff(1) to report on what objects changed since the last build. 51# 52# For patch deliverable purposes, it's advised to have nightly do a clobber, 53# non-debug build. 54# 55# Think about the results. Was something flagged that you don't expect? Go look 56# at the results file to see details about the differences. 57# 58# Use the -i option in conjunction with -v and -V to dive deeper and have 59# wsdiff(1) report with more verbosity. 60# 61# Usage: wsdiff [-dstuvV] [-U lines] [-r results ] [-i filelist ] old new 62# 63# Where "old" is the path to the proto area build without the changes, and 64# "new" is the path to the proto area built with the changes. The following 65# options are supported: 66# 67# -d Print debug messages about the progress 68# -i Tell wsdiff which objects to compare via an input file list 69# -r Log results and observed differences 70# -s Produce sorted list of differences 71# -t Use onbld tools in $SRC/tools 72# -u Produce unified diff output 73# -U Produce unified diff output with <lines> lines of context 74# -v Do not truncate observed diffs in results 75# -V Log *all* ELF sect diffs vs. logging the first diff found 76 77from __future__ import print_function 78import datetime, fnmatch, getopt, os, profile, io, subprocess 79import re, resource, select, shutil, signal, string, struct, sys, tempfile 80import time, threading 81from stat import * 82 83PY3 = sys.version_info[0] == 3 84 85# Human readable diffs truncated by default if longer than this 86# Specifying -v on the command line will override 87diffs_sz_thresh = 4096 88diff_args = '' 89 90# Lock name Provides exclusive access to 91# --------------+------------------------------------------------ 92# output_lock standard output or temporary file (difference()) 93# log_lock the results file (log_difference()) 94# wset_lock changedFiles list (workerThread()) 95output_lock = threading.Lock() 96log_lock = threading.Lock() 97wset_lock = threading.Lock() 98 99# Variable for thread control 100keep_processing = True 101 102# Default search path for wsdiff 103wsdiff_path = [ "/usr/bin", 104 "/usr/ccs/bin", 105 "/lib/svc/bin", 106 "/opt/onbld/bin" ] 107 108# These are objects that wsdiff will notice look different, but will not report. 109# Existence of an exceptions list, and adding things here is *dangerous*, 110# and therefore the *only* reasons why anything would be listed here is because 111# the objects do not build deterministically, yet we *cannot* fix this. 112# 113wsdiff_exceptions = [ 114] 115 116# Path to genunix, used for finding the proto root, and for CTF diff 117genunix = "kernel/amd64/genunix" 118 119if PY3: 120 def getoutput(cmd): 121 import shlex, tempfile 122 f, fpath = tempfile.mkstemp() 123 status = os.system("{ " + cmd + "; } >" + 124 shlex.quote(fpath) + " 2>&1") 125 returncode = os.WEXITSTATUS(status) 126 with os.fdopen(f, mode="r", errors="ignore") as tfile: 127 output = tfile.read() 128 os.unlink(fpath) 129 if output[-1:] == '\n': 130 output = output[:-1] 131 return returncode, output 132else: 133 import commands 134 getoutput = commands.getstatusoutput 135 136##### 137# Logging routines 138# 139 140# Debug message to be printed to the screen, and the log file 141def debug(msg) : 142 143 # Add prefix to highlight debugging message 144 msg = "## " + msg 145 if debugon : 146 output_lock.acquire() 147 print(msg) 148 sys.stdout.flush() 149 output_lock.release() 150 if logging : 151 log_lock.acquire() 152 print(msg, file=log) 153 log.flush() 154 log_lock.release() 155 156# Informational message to be printed to the screen, and the log file 157def info(msg) : 158 159 output_lock.acquire() 160 print(msg) 161 sys.stdout.flush() 162 output_lock.release() 163 if logging : 164 log_lock.acquire() 165 print(msg, file=log) 166 log.flush() 167 log_lock.release() 168 169# Error message to be printed to the screen, and the log file 170def error(msg) : 171 172 output_lock.acquire() 173 print("ERROR: " + msg, file=sys.stderr) 174 sys.stderr.flush() 175 output_lock.release() 176 if logging : 177 log_lock.acquire() 178 print("ERROR: " + msg, file=log) 179 log.flush() 180 log_lock.release() 181 182# Informational message to be printed only to the log, if there is one. 183def v_info(msg) : 184 185 if logging : 186 log_lock.acquire() 187 print(msg, file=log) 188 log.flush() 189 log_lock.release() 190 191# 192# Flag a detected file difference 193# Display the fileName to stdout, and log the difference 194# 195def difference(f, dtype, diffs) : 196 197 if f in wsdiff_exceptions : 198 return 199 200 output_lock.acquire() 201 if o_sorted : 202 differentFiles.append(f) 203 else: 204 print(f) 205 sys.stdout.flush() 206 output_lock.release() 207 208 log_difference(f, dtype, diffs) 209 210# 211# Do the actual logging of the difference to the results file 212# 213def log_difference(f, dtype, diffs) : 214 215 if logging : 216 log_lock.acquire() 217 print(f, file=log) 218 print("NOTE: " + dtype + " difference detected.", file=log) 219 220 difflen = len(diffs) 221 if difflen > 0 : 222 print('', file=log) 223 224 if not vdiffs and difflen > diffs_sz_thresh : 225 print(diffs[:diffs_sz_thresh], file=log) 226 print("... truncated due to length: " + 227 "use -v to override ...", file=log) 228 else : 229 print(diffs, file=log) 230 print('\n', file=log) 231 log.flush() 232 log_lock.release() 233 234 235##### 236# diff generating routines 237# 238 239# 240# Return human readable diffs from two files 241# 242def diffFileData(tmpf1, tmpf2) : 243 244 binaries = False 245 246 # Filter the data through od(1) if the data is detected 247 # as being binary 248 if isBinary(tmpf1) or isBinary(tmpf2) : 249 binaries = True 250 tmp_od1 = tmpf1 + ".od" 251 tmp_od2 = tmpf2 + ".od" 252 253 cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1 254 os.system(cmd) 255 cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2 256 os.system(cmd) 257 258 tmpf1 = tmp_od1 259 tmpf2 = tmp_od2 260 261 dcmd = "{} {} {} {}".format(diff_cmd, diff_args, tmpf1, tmpf2) 262 try: 263 rc, data = getoutput(dcmd) 264 if rc == 0: 265 # No differences found 266 data = '' 267 # If producing unified output, strip the first two lines 268 # which just show the temporary file names. 269 if diff_args: 270 data = data.split("\n", 2)[-1] 271 272 # Remove the temp files as we no longer need them. 273 if binaries : 274 try: 275 os.unlink(tmp_od1) 276 os.unlink(tmp_od2) 277 except OSError as e: 278 error("diffFileData: unlink failed {}" 279 .format(e)) 280 except: 281 error("failed to get output of command: " + dcmd) 282 283 # Send exception for the failed command up 284 raise 285 return 286 287 return data 288 289##### 290# Misc utility functions 291# 292 293# Prune off the leading prefix from string s 294def str_prefix_trunc(s, prefix) : 295 snipLen = len(prefix) 296 return s[snipLen:] 297 298# 299# Prune off leading proto path goo (if there is one) to yield 300# the deliverable's eventual path relative to root 301# e.g. proto.base/root_sparc/usr/src/cmd/prstat => usr/src/cmd/prstat 302# 303def fnFormat(fn) : 304 global baseWsRoot, ptchWsRoot 305 306 if (baseWsRoot and 307 len(os.path.commonprefix([fn, baseWsRoot])) == len(baseWsRoot)): 308 return fn[len(baseWsRoot) + 1:] 309 310 if (ptchWsRoot and 311 len(os.path.commonprefix([fn, ptchWsRoot])) == len(ptchWsRoot)): 312 return fn[len(ptchWsRoot) + 1:] 313 314 # Fall back to looking for the expected root_<arch>[-nd] string 315 316 pos = fn.find("root_" + arch) 317 if pos == -1 : 318 return fn 319 320 pos = fn.find("/", pos) 321 if pos == -1 : 322 return fn 323 324 return fn[pos + 1:] 325 326# 327# Find the path to a proto root given the name of a file or directory under it 328# e.g. proto.base/root_i386-nd/usr/bin => proto.base/root_i386-nd 329# root/usr/bin => root 330# 331def protoroot(fn): 332 root = fn.rstrip('/') 333 while len(root) > 1: 334 if os.path.isfile(os.path.join(root, genunix)): 335 return root 336 root = os.path.dirname(root) 337 338 # genunix was not found, try and determine the root by checking for 339 # the expected root_<arch>[-nd] string 340 341 pos = fn.find("root_" + arch) 342 if pos == -1: 343 return None 344 345 pos = fn.find("/", pos) 346 if pos == -1: 347 return fn 348 349 return fn[:pos] 350 351##### 352# Usage / argument processing 353# 354 355# 356# Display usage message 357# 358def usage() : 359 sys.stdout.flush() 360 print("""Usage: wsdiff [-dstuvV] [-U lines] [-r results ] [-i filelist ] old new 361 -d Print debug messages about the progress 362 -i Tell wsdiff which objects to compare via an input file list 363 -r Log results and observed differences 364 -s Produce sorted list of differences 365 -t Use onbld tools in $SRC/tools 366 -u Produce unified diff output 367 -U Produce unified diff output with <lines> lines of context 368 -v Do not truncate observed diffs in results 369 -V Log *all* ELF sect diffs vs. logging the first diff found""", 370 file=sys.stderr) 371 sys.exit(1) 372 373# 374# Process command line options 375# 376def args() : 377 378 global debugon 379 global logging 380 global vdiffs 381 global reportAllSects 382 global o_sorted 383 global diff_args 384 385 validOpts = 'di:r:uU:vVst?' 386 387 baseRoot = "" 388 ptchRoot = "" 389 fileNamesFile = "" 390 results = "" 391 localTools = False 392 393 # getopt.getopt() returns: 394 # an option/value tuple 395 # a list of remaining non-option arguments 396 # 397 # A correct wsdiff invocation will have exactly two non option 398 # arguments, the paths to the base (old), ptch (new) proto areas 399 try: 400 optlist, args = getopt.getopt(sys.argv[1:], validOpts) 401 except getopt.error as val: 402 usage() 403 404 if len(args) != 2 : 405 usage(); 406 407 for opt,val in optlist : 408 if opt == '-d' : 409 debugon = True 410 elif opt == '-i' : 411 fileNamesFile = val 412 elif opt == '-r' : 413 results = val 414 logging = True 415 elif opt == '-s' : 416 o_sorted = True 417 elif opt == '-u' : 418 diff_args = '-u' 419 elif opt == '-U' : 420 diff_args = '-U' + str(val) 421 elif opt == '-v' : 422 vdiffs = True 423 elif opt == '-V' : 424 reportAllSects = True 425 elif opt == '-t': 426 localTools = True 427 else: 428 usage() 429 430 baseRoot = args[0] 431 ptchRoot = args[1] 432 433 if len(baseRoot) == 0 or len(ptchRoot) == 0 : 434 usage() 435 436 if logging and len(results) == 0 : 437 usage() 438 439 if vdiffs and not logging : 440 error("The -v option requires a results file (-r)") 441 sys.exit(1) 442 443 if reportAllSects and not logging : 444 error("The -V option requires a results file (-r)") 445 sys.exit(1) 446 447 # alphabetical order 448 return baseRoot, fileNamesFile, localTools, ptchRoot, results 449 450##### 451# File identification 452# 453 454# 455# Identify the file type. 456# If it's not ELF, use the file extension to identify 457# certain file types that require special handling to 458# compare. Otherwise just return a basic "ASCII" type. 459# 460def getTheFileType(f) : 461 462 extensions = { 'a' : 'ELF Object Archive', 463 'jar' : 'Java Archive', 464 'html' : 'HTML', 465 'ln' : 'Lint Library', 466 'db' : 'Sqlite Database' } 467 468 try: 469 if os.stat(f)[ST_SIZE] == 0 : 470 return 'ASCII' 471 except: 472 error("failed to stat " + f) 473 return 'Error' 474 475 if isELF(f) == 1 : 476 return 'ELF' 477 478 fnamelist = f.split('.') 479 if len(fnamelist) > 1 : # Test the file extension 480 extension = fnamelist[-1] 481 if extension in extensions.keys(): 482 return extensions[extension] 483 484 return 'ASCII' 485 486# 487# Return non-zero if "f" is an ELF file 488# 489elfmagic = b'\177ELF' 490def isELF(f) : 491 try: 492 with open(f, mode='rb') as fd: 493 magic = fd.read(len(elfmagic)) 494 495 if magic == elfmagic : 496 return 1 497 except: 498 pass 499 return 0 500 501# 502# Return non-zero is "f" is binary. 503# Consider the file to be binary if it contains any null characters 504# 505def isBinary(f) : 506 try: 507 with open(f, mode='rb') as fd: 508 s = fd.read() 509 510 if s.find(b'\0') == -1 : 511 return 0 512 except: 513 pass 514 return 1 515 516##### 517# Directory traversal and file finding 518# 519 520# 521# Return a sorted list of files found under the specified directory 522# 523def findFiles(d) : 524 for path, subdirs, files in os.walk(d) : 525 files.sort() 526 for name in files : 527 yield os.path.join(path, name) 528 529# 530# Examine all files in base, ptch 531# 532# Return a list of files appearing in both proto areas, 533# a list of new files (files found only in ptch) and 534# a list of deleted files (files found only in base) 535# 536def protoCatalog(base, ptch) : 537 538 compFiles = [] # List of files in both proto areas 539 ptchList = [] # List of file in patch proto area 540 541 newFiles = [] # New files detected 542 deletedFiles = [] # Deleted files 543 544 debug("Getting the list of files in the base area"); 545 baseFilesList = list(findFiles(base)) 546 baseStringLength = len(base) 547 debug("Found " + str(len(baseFilesList)) + " files") 548 549 debug("Getting the list of files in the patch area"); 550 ptchFilesList = list(findFiles(ptch)) 551 ptchStringLength = len(ptch) 552 debug("Found " + str(len(ptchFilesList)) + " files") 553 554 # Inventory files in the base proto area 555 debug("Determining the list of regular files in the base area"); 556 for fn in baseFilesList : 557 if os.path.islink(fn) : 558 continue 559 560 fileName = fn[baseStringLength:] 561 compFiles.append(fileName) 562 debug("Found " + str(len(compFiles)) + " files") 563 564 # Inventory files in the patch proto area 565 debug("Determining the list of regular files in the patch area"); 566 for fn in ptchFilesList : 567 if os.path.islink(fn) : 568 continue 569 570 fileName = fn[ptchStringLength:] 571 ptchList.append(fileName) 572 debug("Found " + str(len(ptchList)) + " files") 573 574 # Deleted files appear in the base area, but not the patch area 575 debug("Searching for deleted files by comparing the lists") 576 for fileName in compFiles : 577 if not fileName in ptchList : 578 deletedFiles.append(fileName) 579 debug("Found " + str(len(deletedFiles)) + " deleted files") 580 581 # Eliminate "deleted" files from the list of objects appearing 582 # in both the base and patch proto areas 583 debug("Eliminating deleted files from the list of objects") 584 for fileName in deletedFiles : 585 try: 586 compFiles.remove(fileName) 587 except: 588 error("filelist.remove() failed") 589 debug("List for comparison reduced to " + str(len(compFiles)) 590 + " files") 591 592 # New files appear in the patch area, but not the base 593 debug("Getting the list of newly added files") 594 for fileName in ptchList : 595 if not fileName in compFiles : 596 newFiles.append(fileName) 597 debug("Found " + str(len(newFiles)) + " new files") 598 599 return compFiles, newFiles, deletedFiles 600 601# 602# Examine the files listed in the input file list 603# 604# Return a list of files appearing in both proto areas, 605# a list of new files (files found only in ptch) and 606# a list of deleted files (files found only in base) 607# 608def flistCatalog(base, ptch, flist) : 609 compFiles = [] # List of files in both proto areas 610 newFiles = [] # New files detected 611 deletedFiles = [] # Deleted files 612 613 try: 614 fd = open(flist, "r") 615 except: 616 error("could not open: " + flist) 617 cleanup(1) 618 619 files = [] 620 files = fd.readlines() 621 fd.close() 622 623 for f in files : 624 ptch_present = True 625 base_present = True 626 627 if f == '\n' : 628 continue 629 630 # the fileNames have a trailing '\n' 631 f = f.rstrip() 632 633 # The objects in the file list have paths relative 634 # to $ROOT or to the base/ptch directory specified on 635 # the command line. 636 # If it's relative to $ROOT, we'll need to add back the 637 # root_`uname -p` goo we stripped off in fnFormat() 638 if os.path.exists(base + f) : 639 fn = f; 640 elif os.path.exists(base + "root_" + arch + "/" + f) : 641 fn = "root_" + arch + "/" + f 642 else : 643 base_present = False 644 645 if base_present : 646 if not os.path.exists(ptch + fn) : 647 ptch_present = False 648 else : 649 if os.path.exists(ptch + f) : 650 fn = f 651 elif os.path.exists(ptch + "root_" + arch + "/" + f) : 652 fn = "root_" + arch + "/" + f 653 else : 654 ptch_present = False 655 656 if os.path.islink(base + fn) : # ignore links 657 base_present = False 658 if os.path.islink(ptch + fn) : 659 ptch_present = False 660 661 if base_present and ptch_present : 662 compFiles.append(fn) 663 elif base_present : 664 deletedFiles.append(fn) 665 elif ptch_present : 666 newFiles.append(fn) 667 else : 668 if (os.path.islink(base + fn) and 669 os.path.islink(ptch + fn)) : 670 continue 671 error(f + " in file list, but not in either tree. " + 672 "Skipping...") 673 674 return compFiles, newFiles, deletedFiles 675 676 677# 678# Build a fully qualified path to an external tool/utility. 679# Consider the default system locations. For onbld tools, if 680# the -t option was specified, we'll try to use built tools in $SRC tools, 681# and otherwise, we'll fall back on /opt/onbld/ 682# 683def find_tool(tool) : 684 685 # First, check what was passed 686 if os.path.exists(tool) : 687 return tool 688 689 # Next try in wsdiff path 690 for pdir in wsdiff_path : 691 location = pdir + "/" + tool 692 if os.path.exists(location) : 693 return location + " " 694 695 location = pdir + "/" + arch + "/" + tool 696 if os.path.exists(location) : 697 return location + " " 698 699 error("Could not find path to: " + tool); 700 sys.exit(1); 701 702 703##### 704# ELF file comparison helper routines 705# 706 707# 708# Return a dictionary of ELF section types keyed by section name 709# 710def get_elfheader(f) : 711 712 header = {} 713 714 rc, hstring = getoutput(elfdump_cmd + " -c " + f) 715 716 if len(hstring) == 0 : 717 error("Failed to dump ELF header for " + f) 718 raise 719 return 720 721 # elfdump(1) dumps the section headers with the section name 722 # following "sh_name:", and the section type following "sh_type:" 723 sections = hstring.split("Section Header") 724 for sect in sections : 725 datap = sect.find("sh_name:"); 726 if datap == -1 : 727 continue 728 section = sect[datap:].split()[1] 729 datap = sect.find("sh_type:"); 730 if datap == -1 : 731 error("Could not get type for sect: " + section + 732 " in " + f) 733 sh_type = sect[datap:].split()[2] 734 header[section] = sh_type 735 736 return header 737 738# 739# Extract data in the specified ELF section from the given file 740# 741def extract_elf_section(f, section) : 742 743 rc, data = getoutput(dump_cmd + " -sn " + section + " " + f) 744 745 if len(data) == 0 : 746 error(dump_cmd + "yielded no data on section " + section + 747 " of " + f) 748 raise 749 return 750 751 # dump(1) displays the file name to start... 752 # get past it to the data itself 753 dbegin = data.find(":") + 1 754 data = data[dbegin:]; 755 756 return (data) 757 758# 759# Return a (hopefully meaningful) human readable set of diffs 760# for the specified ELF section between f1 and f2 761# 762# Depending on the section, various means for dumping and diffing 763# the data may be employed. 764# 765 766text_sections = [ '.text', '.init', '.fini' ] 767 768# Helper to generate the requireed commands for diffing two .SUNW_ctf 769# sections. 770def diff_ctf(f1, f2): 771 772 # Find genunix so that it can be used for parent CTF data when 773 # appropriate. 774 if diff_ctf.genunix1 is None: 775 global genunix, baseWsRoot, ptchWsRoot 776 777 if (baseWsRoot and ptchWsRoot and 778 os.path.isfile(os.path.join(baseWsRoot, genunix)) and 779 os.path.isfile(os.path.join(ptchWsRoot, genunix))): 780 diff_ctf.genunix1 = os.path.join(baseWsRoot, genunix) 781 diff_ctf.genunix2 = os.path.join(ptchWsRoot, genunix) 782 debug("CTF: Found {}".format(diff_ctf.genunix1)) 783 debug("CTF: Found {}".format(diff_ctf.genunix2)) 784 else: 785 # Could not find genunix, do the best we can. 786 error("diff_ctf: Could not find genunix. " + 787 "CTF diffs will be less useful.") 788 diff_ctf.genunix1 = diff_ctf.genunix2 = False 789 790 # Determine if this is a merged file from genunix by looking 791 # at the parent 792 rc, data = getoutput("{} -h {}".format(ctfdump_cmd, f1)) 793 if rc != 0: 794 error("Could not read CTF header: {}".format(data)) 795 return (None, None) 796 797 parent = None 798 for line in data.split('\n'): 799 if line.strip().startswith('cth_parname'): 800 try: 801 parent = line.split('=')[1].strip() 802 break 803 except: 804 pass 805 806 cmd1 = cmd2 = "{} -c ".format(ctfdump_cmd) 807 if parent == "genunix": 808 if diff_ctf.genunix1 and diff_ctf.genunix2: 809 cmd1 += "-p {} ".format(diff_ctf.genunix1) 810 cmd2 += "-p {} ".format(diff_ctf.genunix2) 811 elif parent is None or (len(parent) > 0 and parent != "(anon)"): 812 error("Unknown CTF Parent: {}".format(parent)) 813 return (None, None) 814 815 cmd1 += f1 816 cmd2 += f2 817 818 return (cmd1, cmd2) 819 820diff_ctf.genunix1 = None 821diff_ctf.genunix2 = None 822 823def diff_elf_section(f1, f2, section, sh_type) : 824 825 t = threading.current_thread() 826 tmpFile1 = tmpDir1 + os.path.basename(f1) + t.name 827 tmpFile2 = tmpDir2 + os.path.basename(f2) + t.name 828 829 if ((sh_type == "SHT_RELA") or 830 (sh_type == "SHT_REL")): # relocation section 831 cmd1 = elfdump_cmd + " -r " + f1 + " > " + tmpFile1 832 cmd2 = elfdump_cmd + " -r " + f2 + " > " + tmpFile2 833 elif (section == ".group") : 834 cmd1 = elfdump_cmd + " -g " + f1 + " > " + tmpFile1 835 cmd2 = elfdump_cmd + " -g " + f2 + " > " + tmpFile2 836 elif (section == ".hash") : 837 cmd1 = elfdump_cmd + " -h " + f1 + " > " + tmpFile1 838 cmd2 = elfdump_cmd + " -h " + f2 + " > " + tmpFile2 839 elif (section == ".dynamic") : 840 cmd1 = elfdump_cmd + " -d " + f1 + " > " + tmpFile1 841 cmd2 = elfdump_cmd + " -d " + f2 + " > " + tmpFile2 842 elif (section == ".got") : 843 cmd1 = elfdump_cmd + " -G " + f1 + " > " + tmpFile1 844 cmd2 = elfdump_cmd + " -G " + f2 + " > " + tmpFile2 845 elif (section == ".SUNW_cap") : 846 cmd1 = elfdump_cmd + " -H " + f1 + " > " + tmpFile1 847 cmd2 = elfdump_cmd + " -H " + f2 + " > " + tmpFile2 848 elif (section == ".interp") : 849 cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1 850 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2 851 elif (section == ".symtab" or section == ".dynsym") : 852 cmd1 = (elfdump_cmd + " -s -N " + section + " " + f1 + 853 " > " + tmpFile1) 854 cmd2 = (elfdump_cmd + " -s -N " + section + " " + f2 + 855 " > " + tmpFile2) 856 elif (section in text_sections) : 857 # dis sometimes complains when it hits something it doesn't 858 # know how to disassemble. Just ignore it, as the output 859 # being generated here is human readable, and we've already 860 # correctly flagged the difference. 861 cmd1 = (dis_cmd + " -t " + section + " " + f1 + 862 " 2>/dev/null | grep -v disassembly > " + tmpFile1) 863 cmd2 = (dis_cmd + " -t " + section + " " + f2 + 864 " 2>/dev/null | grep -v disassembly > " + tmpFile2) 865 elif (section == ".SUNW_ctf"): 866 (cmd1, cmd2) = diff_ctf(f1, f2) 867 if not cmd1: 868 return "" 869 cmd1 += " > {}".format(tmpFile1) 870 cmd2 += " > {}".format(tmpFile2) 871 872 else : 873 cmd1 = (elfdump_cmd + " -w " + tmpFile1 + " -N " + 874 section + " " + f1) 875 cmd2 = (elfdump_cmd + " -w " + tmpFile2 + " -N " + 876 section + " " + f2) 877 878 os.system(cmd1) 879 os.system(cmd2) 880 881 data = diffFileData(tmpFile1, tmpFile2) 882 883 # remove temp files as we no longer need them 884 try: 885 os.unlink(tmpFile1) 886 os.unlink(tmpFile2) 887 except OSError as e: 888 error("diff_elf_section: unlink failed {}".format(e)) 889 890 return (data) 891 892# 893# compare the relevant sections of two ELF binaries 894# and report any differences 895# 896# Returns: 1 if any differenes found 897# 0 if no differences found 898# -1 on error 899# 900 901# Sections deliberately not considered when comparing two ELF 902# binaries. Differences observed in these sections are not considered 903# significant where patch deliverable identification is concerned. 904sections_to_skip = [ ".SUNW_signature", 905 ".comment", 906 ".debug", 907 ".plt", 908 ".rela.bss", 909 ".rela.plt", 910 ".line", 911 ".note", 912 ".compcom", 913 ".SUNW_dof", 914 ] 915 916sections_preferred = [ ".SUNW_ctf", 917 ".rodata.str1.8", 918 ".rodata.str1.1", 919 ".rodata", 920 ".data1", 921 ".data", 922 ".text", 923 ] 924 925# Some sections must always be extracted and diffed to check that there are 926# real differences. 927sections_to_always_diff = [ ".SUNW_ctf" ] 928 929def compareElfs(base, ptch, quiet) : 930 931 global logging 932 933 try: 934 base_header = get_elfheader(base) 935 except: 936 return 937 sections = list(base_header.keys()) 938 sections.sort() 939 940 try: 941 ptch_header = get_elfheader(ptch) 942 except: 943 return 944 e2_only_sections = list(ptch_header.keys()) 945 e2_only_sections.sort() 946 947 e1_only_sections = [] 948 949 fileName = fnFormat(base) 950 951 # Derive the list of ELF sections found only in 952 # either e1 or e2. 953 for sect in sections : 954 if not sect in e2_only_sections : 955 e1_only_sections.append(sect) 956 else : 957 e2_only_sections.remove(sect) 958 959 if len(e1_only_sections) > 0 : 960 if quiet : 961 return 1 962 963 data = "" 964 if logging : 965 slist = "" 966 for sect in e1_only_sections : 967 slist = slist + sect + "\t" 968 data = ("ELF sections found in " + 969 base + " but not in " + ptch + 970 "\n\n" + slist) 971 972 difference(fileName, "ELF", data) 973 return 1 974 975 if len(e2_only_sections) > 0 : 976 if quiet : 977 return 1 978 979 data = "" 980 if logging : 981 slist = "" 982 for sect in e2_only_sections : 983 slist = slist + sect + "\t" 984 data = ("ELF sections found in " + 985 ptch + " but not in " + base + 986 "\n\n" + slist) 987 988 difference(fileName, "ELF", data) 989 return 1 990 991 # Look for preferred sections, and put those at the 992 # top of the list of sections to compare 993 for psect in sections_preferred : 994 if psect in sections : 995 sections.remove(psect) 996 sections.insert(0, psect) 997 998 # Compare ELF sections 999 first_section = True 1000 for sect in sections : 1001 1002 if sect in sections_to_skip : 1003 continue 1004 1005 try: 1006 s1 = extract_elf_section(base, sect); 1007 except: 1008 return 1009 1010 try: 1011 s2 = extract_elf_section(ptch, sect); 1012 except: 1013 return 1014 1015 if len(s1) != len (s2) or s1 != s2: 1016 if not quiet or sect in sections_to_always_diff: 1017 sh_type = base_header[sect] 1018 data = diff_elf_section(base, ptch, 1019 sect, sh_type) 1020 1021 if len(data) == 0: 1022 continue # No differences 1023 1024 if not quiet: 1025 # If all ELF sections are being reported, then 1026 # invoke difference() to flag the file name to 1027 # stdout only once. Any other section 1028 # differences should be logged to the results 1029 # file directly 1030 if not first_section : 1031 log_difference(fileName, 1032 "ELF " + sect, data) 1033 else : 1034 difference(fileName, "ELF " + sect, 1035 data) 1036 1037 if not reportAllSects : 1038 return 1 1039 first_section = False 1040 1041 return 0 1042 1043##### 1044# recursively remove 2 directories 1045# 1046# Used for removal of temporary directory strucures (ignores any errors). 1047# 1048def clearTmpDirs(dir1, dir2) : 1049 1050 if os.path.isdir(dir1) > 0 : 1051 shutil.rmtree(dir1, True) 1052 1053 if os.path.isdir(dir2) > 0 : 1054 shutil.rmtree(dir2, True) 1055 1056 1057##### 1058# Archive object comparison 1059# 1060# Returns 1 if difference detected 1061# 0 if no difference detected 1062# -1 on error 1063# 1064def compareArchives(base, ptch, fileType) : 1065 1066 fileName = fnFormat(base) 1067 t = threading.current_thread() 1068 ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.name 1069 ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.name 1070 1071 # 1072 # Be optimistic and first try a straight file compare 1073 # as it will allow us to finish up quickly. 1074 # 1075 if compareBasic(base, ptch, True, fileType) == 0 : 1076 return 0 1077 1078 try: 1079 os.makedirs(ArchTmpDir1) 1080 os.makedirs(ArchTmpDir2) 1081 except OSError as e: 1082 error("compareArchives: makedir failed {}".format(e)) 1083 return -1 1084 1085 # copy over the objects to the temp areas, and 1086 # unpack them 1087 baseCmd = "cp -fp " + base + " " + ArchTmpDir1 1088 rc, output = getoutput(baseCmd) 1089 if rc != 0: 1090 error(baseCmd + " failed: " + output) 1091 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1092 return -1 1093 1094 ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2 1095 rc, output = getoutput(ptchCmd) 1096 if rc != 0: 1097 error(ptchCmd + " failed: " + output) 1098 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1099 return -1 1100 1101 bname = fileName.split('/')[-1] 1102 if fileType == "Java Archive" : 1103 baseCmd = ("cd " + ArchTmpDir1 + "; " + "jar xf " + bname + 1104 "; rm -f " + bname + " META-INF/MANIFEST.MF") 1105 ptchCmd = ("cd " + ArchTmpDir2 + "; " + "jar xf " + bname + 1106 "; rm -f " + bname + " META-INF/MANIFEST.MF") 1107 elif fileType == "ELF Object Archive" : 1108 baseCmd = ("cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + 1109 bname + "; rm -f " + bname) 1110 ptchCmd = ("cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + 1111 bname + "; rm -f " + bname) 1112 else : 1113 error("unexpected file type: " + fileType) 1114 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1115 return -1 1116 1117 os.system(baseCmd) 1118 os.system(ptchCmd) 1119 1120 baseFlist = list(findFiles(ArchTmpDir1)) 1121 ptchFlist = list(findFiles(ArchTmpDir2)) 1122 1123 # Trim leading path off base/ptch file lists 1124 flist = [] 1125 for fn in baseFlist : 1126 flist.append(str_prefix_trunc(fn, ArchTmpDir1)) 1127 baseFlist = flist 1128 1129 flist = [] 1130 for fn in ptchFlist : 1131 flist.append(str_prefix_trunc(fn, ArchTmpDir2)) 1132 ptchFlist = flist 1133 1134 for fn in ptchFlist : 1135 if not fn in baseFlist : 1136 difference(fileName, fileType, 1137 fn + " added to " + fileName) 1138 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1139 return 1 1140 1141 for fn in baseFlist : 1142 if not fn in ptchFlist : 1143 difference(fileName, fileType, 1144 fn + " removed from " + fileName) 1145 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1146 return 1 1147 1148 differs = compareOneFile((ArchTmpDir1 + fn), 1149 (ArchTmpDir2 + fn), True) 1150 if differs : 1151 difference(fileName, fileType, 1152 fn + " in " + fileName + " differs") 1153 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1154 return 1 1155 1156 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1157 return 0 1158 1159##### 1160# (Basic) file comparison 1161# 1162# Returns 1 if difference detected 1163# 0 if no difference detected 1164# -1 on error 1165# 1166def compareBasic(base, ptch, quiet, fileType) : 1167 1168 fileName = fnFormat(base); 1169 1170 if quiet and os.stat(base)[ST_SIZE] != os.stat(ptch)[ST_SIZE] : 1171 return 1 1172 1173 try: 1174 with open(base, 'rb') as fh: 1175 baseData = fh.read() 1176 except: 1177 error("could not open " + base) 1178 return -1 1179 1180 try: 1181 with open(ptch, 'rb') as fh: 1182 ptchData = fh.read() 1183 except: 1184 error("could not open " + ptch) 1185 return -1 1186 1187 if quiet : 1188 if baseData != ptchData : 1189 return 1 1190 else : 1191 if len(baseData) != len(ptchData) or baseData != ptchData : 1192 diffs = diffFileData(base, ptch) 1193 difference(fileName, fileType, diffs) 1194 return 1 1195 return 0 1196 1197 1198##### 1199# Compare two objects by producing a data dump from 1200# each object, and then comparing the dump data 1201# 1202# Returns: 1 if a difference is detected 1203# 0 if no difference detected 1204# -1 upon error 1205# 1206def compareByDumping(base, ptch, quiet, fileType) : 1207 1208 fileName = fnFormat(base); 1209 t = threading.current_thread() 1210 tmpFile1 = tmpDir1 + os.path.basename(base) + t.name 1211 tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.name 1212 1213 if fileType == "Lint Library" : 1214 baseCmd = (lintdump_cmd + " -ir " + base + 1215 " | egrep -v '(LINTOBJ|LINTMOD):'" + 1216 " | grep -v PASS[1-3]:" + 1217 " > " + tmpFile1) 1218 ptchCmd = (lintdump_cmd + " -ir " + ptch + 1219 " | egrep -v '(LINTOBJ|LINTMOD):'" + 1220 " | grep -v PASS[1-3]:" + 1221 " > " + tmpFile2) 1222 elif fileType == "Sqlite Database" : 1223 baseCmd = ("echo .dump | " + sqlite_cmd + base + " > " + 1224 tmpFile1) 1225 ptchCmd = ("echo .dump | " + sqlite_cmd + ptch + " > " + 1226 tmpFile2) 1227 1228 os.system(baseCmd) 1229 os.system(ptchCmd) 1230 1231 try: 1232 with open(tmpFile1, 'rb') as fh: 1233 baseData = fh.read() 1234 except: 1235 error("could not open: " + tmpFile1) 1236 return 1237 1238 try: 1239 with open(tmpFile2, 'rb') as fh: 1240 ptchData = fh.read() 1241 except: 1242 error("could not open: " + tmpFile2) 1243 return 1244 1245 ret = 0 1246 1247 if len(baseData) != len(ptchData) or baseData != ptchData : 1248 if not quiet : 1249 data = diffFileData(tmpFile1, tmpFile2); 1250 ret = 1 1251 1252 # Remove the temporary files now. 1253 try: 1254 os.unlink(tmpFile1) 1255 os.unlink(tmpFile2) 1256 except OSError as e: 1257 error("compareByDumping: unlink failed {}".format(e)) 1258 1259 return ret 1260 1261##### 1262# 1263# SIGINT signal handler. Changes thread control variable to tell the threads 1264# to finish their current job and exit. 1265# 1266def discontinue_processing(signl, frme): 1267 global keep_processing 1268 1269 print("Caught Ctrl-C, stopping the threads", file=sys.stderr) 1270 keep_processing = False 1271 1272 return 0 1273 1274##### 1275# 1276# worker thread for changedFiles processing 1277# 1278class workerThread(threading.Thread) : 1279 def run(self): 1280 global wset_lock 1281 global changedFiles 1282 global baseRoot 1283 global ptchRoot 1284 global keep_processing 1285 1286 while (keep_processing) : 1287 # grab the lock to changedFiles and remove one member 1288 # and process it 1289 wset_lock.acquire() 1290 try : 1291 fn = changedFiles.pop() 1292 except IndexError : 1293 # there is nothing more to do 1294 wset_lock.release() 1295 return 1296 wset_lock.release() 1297 1298 base = baseRoot + fn 1299 ptch = ptchRoot + fn 1300 1301 compareOneFile(base, ptch, False) 1302 1303 1304##### 1305# Compare two objects. Detect type changes. 1306# Vector off to the appropriate type specific 1307# compare routine based on the type. 1308# 1309def compareOneFile(base, ptch, quiet) : 1310 1311 # Verify the file types. 1312 # If they are different, indicate this and move on 1313 btype = getTheFileType(base) 1314 ptype = getTheFileType(ptch) 1315 1316 if btype == 'Error' or ptype == 'Error' : 1317 return -1 1318 1319 fileName = fnFormat(base) 1320 1321 if (btype != ptype) : 1322 if not quiet : 1323 difference(fileName, "file type", 1324 btype + " to " + ptype) 1325 return 1 1326 else : 1327 fileType = btype 1328 1329 if (fileType == 'ELF') : 1330 return compareElfs(base, ptch, quiet) 1331 1332 elif (fileType == 'Java Archive' or fileType == 'ELF Object Archive') : 1333 return compareArchives(base, ptch, fileType) 1334 1335 elif (fileType == 'HTML') : 1336 return compareBasic(base, ptch, quiet, fileType) 1337 1338 elif ( fileType == 'Lint Library' ) : 1339 return compareByDumping(base, ptch, quiet, fileType) 1340 1341 elif ( fileType == 'Sqlite Database' ) : 1342 return compareByDumping(base, ptch, quiet, fileType) 1343 1344 else : 1345 # it has to be some variety of text file 1346 return compareBasic(base, ptch, quiet, fileType) 1347 1348# Cleanup and self-terminate 1349def cleanup(ret) : 1350 1351 debug("Performing cleanup (" + str(ret) + ")") 1352 if os.path.isdir(tmpDir1) > 0 : 1353 shutil.rmtree(tmpDir1) 1354 1355 if os.path.isdir(tmpDir2) > 0 : 1356 shutil.rmtree(tmpDir2) 1357 1358 if logging : 1359 log.close() 1360 1361 sys.exit(ret) 1362 1363def main() : 1364 1365 # Log file handle 1366 global log 1367 1368 # Globals relating to command line options 1369 global logging, vdiffs, reportAllSects 1370 1371 # Named temporary files / directories 1372 global tmpDir1, tmpDir2 1373 1374 # Command paths 1375 global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, \ 1376 diff_cmd, sqlite_cmd, ctfdump_cmd 1377 1378 # Default search path 1379 global wsdiff_path 1380 1381 # Essentially "uname -p" 1382 global arch 1383 1384 # changed files for worker thread processing 1385 global changedFiles 1386 1387 global baseRoot, ptchRoot, baseWsRoot, ptchWsRoot 1388 1389 # Sort the list of files from a temporary file 1390 global o_sorted 1391 global differentFiles 1392 1393 # Debugging indicator 1394 global debugon 1395 1396 # Some globals need to be initialized 1397 debugon = logging = vdiffs = reportAllSects = o_sorted = False 1398 1399 # Process command line arguments 1400 # Return values are returned from args() in alpha order 1401 # (Yes, python functions can return multiple values (ewww)) 1402 # Note that args() also set the globals: 1403 # logging to True if verbose logging (to a file) was enabled 1404 # vdiffs to True if logged differences aren't to be truncated 1405 # reportAllSects to True if all ELF section differences are to 1406 # be reported 1407 # 1408 baseRoot, fileNamesFile, localTools, ptchRoot, results = args() 1409 1410 # 1411 # Set up the results/log file 1412 # 1413 if logging : 1414 try: 1415 log = open(results, "w") 1416 except: 1417 logging = False 1418 error("failed to open log file: {}".format(log)) 1419 sys.exit(1) 1420 1421 v_info("# This file was produced by wsdiff") 1422 dateTimeStr= time.strftime('# %Y-%m-%d at %H:%M:%S', 1423 time.localtime()) 1424 v_info(dateTimeStr) 1425 1426 # Changed files (used only for the sorted case) 1427 if o_sorted : 1428 differentFiles = [] 1429 1430 # 1431 # Build paths to the tools required tools 1432 # 1433 # Try to look for tools in $SRC/tools if the "-t" option 1434 # was specified 1435 # 1436 rc, arch = getoutput("uname -p") 1437 arch = arch.rstrip() 1438 if localTools : 1439 try: 1440 src = os.environ['SRC'] 1441 except: 1442 error("-t specified, but $SRC not set. " + 1443 "Cannot find $SRC/tools") 1444 src = "" 1445 if len(src) > 0 : 1446 wsdiff_path.insert(0, 1447 src + "/tools/proto/opt/onbld/bin") 1448 1449 lintdump_cmd = find_tool("lintdump") 1450 elfdump_cmd = find_tool("elfdump") 1451 dump_cmd = find_tool("dump") 1452 od_cmd = find_tool("od") 1453 dis_cmd = find_tool("dis") 1454 diff_cmd = find_tool("diff") 1455 sqlite_cmd = find_tool("sqlite") 1456 ctfdump_cmd = find_tool("ctfdump") 1457 1458 # 1459 # Set resource limit for number of open files as high as possible. 1460 # This might get handy with big number of threads. 1461 # 1462 (nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE) 1463 try: 1464 resource.setrlimit(resource.RLIMIT_NOFILE, 1465 (nofile_hard, nofile_hard)) 1466 except: 1467 error("cannot set resource limits for number of open files") 1468 sys.exit(1) 1469 1470 # 1471 # validate the base and patch paths 1472 # 1473 baseRoot = os.path.abspath(baseRoot) + os.sep 1474 ptchRoot = os.path.abspath(ptchRoot) + os.sep 1475 1476 if not os.path.exists(baseRoot) : 1477 error("old proto area: {} does not exist".format(baseRoot)) 1478 sys.exit(1) 1479 1480 if not os.path.exists(ptchRoot) : 1481 error("new proto area: {} does not exist".format(ptchRoot)) 1482 sys.exit(1) 1483 1484 # 1485 # attempt to find the workspace root directory for the proto area 1486 # 1487 baseWsRoot = protoroot(baseRoot) 1488 ptchWsRoot = protoroot(ptchRoot) 1489 debug("base workspace root: {}".format(baseWsRoot)) 1490 debug("ptch workspace root: {}".format(ptchWsRoot)) 1491 1492 # 1493 # log some information identifying the run 1494 # 1495 v_info("Old proto area: {}".format(baseRoot)) 1496 v_info("New proto area: {}".format(ptchRoot)) 1497 v_info("Results file: {}".format(results)) 1498 v_info("") 1499 1500 # 1501 # Set up the temporary directories / files 1502 # Could use python's tmpdir routines, but these should 1503 # be easier to identify / keep around for debugging 1504 pid = os.getpid() 1505 tmpDir1 = "/tmp/wsdiff_tmp1_" + str(pid) + "/" 1506 tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/" 1507 try: 1508 os.makedirs(tmpDir1) 1509 os.makedirs(tmpDir2) 1510 except OSError as e: 1511 error("main: makedir failed {}".format(e)) 1512 1513 # Derive a catalog of new, deleted, and to-be-compared objects 1514 # either from the specified base and patch proto areas, or from 1515 # from an input file list 1516 newOrDeleted = False 1517 1518 if fileNamesFile != "" : 1519 changedFiles, newFiles, deletedFiles = \ 1520 flistCatalog(baseRoot, ptchRoot, fileNamesFile) 1521 else : 1522 changedFiles, newFiles, deletedFiles = \ 1523 protoCatalog(baseRoot, ptchRoot) 1524 1525 if len(newFiles) > 0 : 1526 newOrDeleted = True 1527 info("\nNew objects found: ") 1528 1529 if o_sorted : 1530 newFiles.sort() 1531 for fn in newFiles : 1532 info(fnFormat(fn)) 1533 1534 if len(deletedFiles) > 0 : 1535 newOrDeleted = True 1536 info("\nObjects removed: ") 1537 1538 if o_sorted : 1539 deletedFiles.sort() 1540 for fn in deletedFiles : 1541 info(fnFormat(fn)) 1542 1543 if newOrDeleted : 1544 info("\nChanged objects: ") 1545 if o_sorted : 1546 debug("The list will appear after the processing is done") 1547 1548 # Here's where all the heavy lifting happens 1549 # Perform a comparison on each object appearing in 1550 # both proto areas. compareOneFile will examine the 1551 # file types of each object, and will vector off to 1552 # the appropriate comparison routine, where the compare 1553 # will happen, and any differences will be reported / logged 1554 1555 # determine maximum number of worker threads by using 1556 # DMAKE_MAX_JOBS environment variable set by nightly(1) 1557 # or get number of CPUs in the system 1558 try: 1559 max_threads = int(os.environ['DMAKE_MAX_JOBS']) 1560 except: 1561 max_threads = os.sysconf("SC_NPROCESSORS_ONLN") 1562 # If we cannot get number of online CPUs in the system 1563 # run unparallelized otherwise bump the number up 20% 1564 # to achieve best results. 1565 if max_threads == -1 : 1566 max_threads = 1 1567 else : 1568 max_threads += int(max_threads/5) 1569 1570 # Set signal handler to attempt graceful exit 1571 debug("Setting signal handler") 1572 signal.signal( signal.SIGINT, discontinue_processing ) 1573 1574 # Create and unleash the threads 1575 # Only at most max_threads must be running at any moment 1576 mythreads = [] 1577 debug("Spawning " + str(max_threads) + " threads"); 1578 for i in range(max_threads) : 1579 thread = workerThread() 1580 mythreads.append(thread) 1581 mythreads[i].start() 1582 1583 # Wait for the threads to finish and do cleanup if interrupted 1584 debug("Waiting for the threads to finish") 1585 while True: 1586 if not True in [thread.is_alive() for thread in mythreads]: 1587 break 1588 else: 1589 # Some threads are still going 1590 time.sleep(1) 1591 1592 # Interrupted by SIGINT 1593 if keep_processing == False : 1594 cleanup(1) 1595 1596 # If the list of differences was sorted it is stored in an array 1597 if o_sorted : 1598 differentFiles.sort() 1599 for f in differentFiles : 1600 info(fnFormat(f)) 1601 1602 # We're done, cleanup. 1603 cleanup(0) 1604 1605if __name__ == '__main__' : 1606 try: 1607 main() 1608 except KeyboardInterrupt : 1609 cleanup(1); 1610