1 #!/usr/bin/python2.6 2 # 3 # CDDL HEADER START 4 # 5 # The contents of this file are subject to the terms of the 6 # Common Development and Distribution License (the "License"). 7 # You may not use this file except in compliance with the License. 8 # 9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 # or http://www.opensolaris.org/os/licensing. 11 # See the License for the specific language governing permissions 12 # and limitations under the License. 13 # 14 # When distributing Covered Code, include this CDDL HEADER in each 15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 # If applicable, add the following below this CDDL HEADER, with the 17 # fields enclosed by brackets "[]" replaced with your own identifying 18 # information: Portions Copyright [yyyy] [name of copyright owner] 19 # 20 # CDDL HEADER END 21 # 22 # Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 # 24 25 # 26 # wsdiff(1) is a tool that can be used to determine which compiled objects 27 # have changed as a result of a given source change. Developers backporting 28 # new features, RFEs and bug fixes need to be able to identify the set of 29 # patch deliverables necessary for feature/fix realization on a patched system. 30 # 31 # The tool works by comparing objects in two trees/proto areas (one build with, 32 # and without the source changes. 33 # 34 # Using wsdiff(1) is fairly simple: 35 # - Bringover to a fresh workspace 36 # - Perform a full non-debug build (clobber if workspace isn't fresh) 37 # - Move the proto area aside, call it proto.old, or something. 38 # - Integrate your changes to the workspace 39 # - Perform another full non-debug clobber build. 40 # - Use wsdiff(1) to see what changed: 41 # $ wsdiff proto.old proto 42 # 43 # By default, wsdiff will print the list of changed objects / deliverables to 44 # stdout. If a results file is specified via -r, the list of differing objects, 45 # and details about why wsdiff(1) thinks they are different will be logged to 46 # the results file. 47 # 48 # By invoking nightly(1) with the -w option to NIGHTLY_FLAGS, nightly(1) will use 49 # wsdiff(1) to report on what objects changed since the last build. 50 # 51 # For patch deliverable purposes, it's advised to have nightly do a clobber, 52 # non-debug build. 53 # 54 # Think about the results. Was something flagged that you don't expect? Go look 55 # at the results file to see details about the differences. 56 # 57 # Use the -i option in conjunction with -v and -V to dive deeper and have wsdiff(1) 58 # report with more verbosity. 59 # 60 # Usage: wsdiff [-vVt] [-r results ] [-i filelist ] old new 61 # 62 # Where "old" is the path to the proto area build without the changes, and 63 # "new" is the path to the proto area built with the changes. The following 64 # options are supported: 65 # 66 # -v Do not truncate observed diffs in results 67 # -V Log *all* ELF sect diffs vs. logging the first diff found 68 # -t Use onbld tools in $SRC/tools 69 # -r Log results and observed differences 70 # -i Tell wsdiff which objects to compare via an input file list 71 72 import datetime, fnmatch, getopt, os, profile, commands 73 import re, resource, select, shutil, signal, string, struct, sys, tempfile 74 import time, threading 75 from stat import * 76 77 # Human readable diffs truncated by default if longer than this 78 # Specifying -v on the command line will override 79 diffs_sz_thresh = 4096 80 81 # Lock name Provides exclusive access to 82 # --------------+------------------------------------------------ 83 # output_lock standard output or temporary file (difference()) 84 # log_lock the results file (log_difference()) 85 # wset_lock changedFiles list (workerThread()) 86 output_lock = threading.Lock() 87 log_lock = threading.Lock() 88 wset_lock = threading.Lock() 89 90 # Variable for thread control 91 keep_processing = True 92 93 # Default search path for wsdiff 94 wsdiff_path = [ "/usr/bin", 95 "/usr/ccs/bin", 96 "/lib/svc/bin", 97 "/opt/onbld/bin" ] 98 99 # These are objects that wsdiff will notice look different, but will not report. 100 # Existence of an exceptions list, and adding things here is *dangerous*, 101 # and therefore the *only* reasons why anything would be listed here is because 102 # the objects do not build deterministically, yet we *cannot* fix this. 103 # 104 # These perl libraries use __DATE__ and therefore always look different. 105 # Ideally, we would purge use the use of __DATE__ from the source, but because 106 # this is source we wish to distribute with Solaris "unchanged", we cannot modify. 107 # 108 wsdiff_exceptions = [ "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1", 109 "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1", 110 "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1", 111 "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1" 112 ] 113 114 ##### 115 # Logging routines 116 # 117 118 # Debug message to be printed to the screen, and the log file 119 def debug(msg) : 120 121 # Add prefix to highlight debugging message 122 msg = "## " + msg 123 if debugon : 124 output_lock.acquire() 125 print >> sys.stdout, msg 126 sys.stdout.flush() 127 output_lock.release() 128 if logging : 129 log_lock.acquire() 130 print >> log, msg 131 log.flush() 132 log_lock.release() 133 134 # Informational message to be printed to the screen, and the log file 135 def info(msg) : 136 137 output_lock.acquire() 138 print >> sys.stdout, msg 139 sys.stdout.flush() 140 output_lock.release() 141 if logging : 142 log_lock.acquire() 143 print >> log, msg 144 log.flush() 145 log_lock.release() 146 147 # Error message to be printed to the screen, and the log file 148 def error(msg) : 149 150 output_lock.acquire() 151 print >> sys.stderr, "ERROR:", msg 152 sys.stderr.flush() 153 output_lock.release() 154 if logging : 155 log_lock.acquire() 156 print >> log, "ERROR:", msg 157 log.flush() 158 log_lock.release() 159 160 # Informational message to be printed only to the log, if there is one. 161 def v_info(msg) : 162 163 if logging : 164 log_lock.acquire() 165 print >> log, msg 166 log.flush() 167 log_lock.release() 168 169 # 170 # Flag a detected file difference 171 # Display the fileName to stdout, and log the difference 172 # 173 def difference(f, dtype, diffs) : 174 175 if f in wsdiff_exceptions : 176 return 177 178 output_lock.acquire() 179 if sorted : 180 differentFiles.append(f) 181 else: 182 print >> sys.stdout, f 183 sys.stdout.flush() 184 output_lock.release() 185 186 log_difference(f, dtype, diffs) 187 188 # 189 # Do the actual logging of the difference to the results file 190 # 191 def log_difference(f, dtype, diffs) : 192 193 if logging : 194 log_lock.acquire() 195 print >> log, f 196 print >> log, "NOTE:", dtype, "difference detected." 197 198 difflen = len(diffs) 199 if difflen > 0 : 200 print >> log 201 202 if not vdiffs and difflen > diffs_sz_thresh : 203 print >> log, diffs[:diffs_sz_thresh] 204 print >> log, \ 205 "... truncated due to length: " \ 206 "use -v to override ..." 207 else : 208 print >> log, diffs 209 print >> log, "\n" 210 log.flush() 211 log_lock.release() 212 213 214 ##### 215 # diff generating routines 216 # 217 218 # 219 # Return human readable diffs from two temporary files 220 # 221 def diffFileData(tmpf1, tmpf2) : 222 223 binaries = False 224 225 # Filter the data through od(1) if the data is detected 226 # as being binary 227 if isBinary(tmpf1) or isBinary(tmpf2) : 228 binaries = True 229 tmp_od1 = tmpf1 + ".od" 230 tmp_od2 = tmpf2 + ".od" 231 232 cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1 233 os.system(cmd) 234 cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2 235 os.system(cmd) 236 237 tmpf1 = tmp_od1 238 tmpf2 = tmp_od2 239 240 try: 241 data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2) 242 # Remove the temp files as we no longer need them. 243 if binaries : 244 try: 245 os.unlink(tmp_od1) 246 except OSError, e: 247 error("diffFileData: unlink failed %s" % e) 248 try: 249 os.unlink(tmp_od2) 250 except OSError, e: 251 error("diffFileData: unlink failed %s" % e) 252 except: 253 error("failed to get output of command: " + diff_cmd + " " \ 254 + tmpf1 + " " + tmpf2) 255 256 # Send exception for the failed command up 257 raise 258 return 259 260 return data 261 262 # 263 # Return human readable diffs betweeen two datasets 264 # 265 def diffData(base, ptch, d1, d2) : 266 267 t = threading.currentThread() 268 tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName() 269 tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() 270 271 try: 272 fd1 = open(tmpFile1, "w") 273 except: 274 error("failed to open: " + tmpFile1) 275 cleanup(1) 276 277 try: 278 fd2 = open(tmpFile2, "w") 279 except: 280 error("failed to open: " + tmpFile2) 281 cleanup(1) 282 283 fd1.write(d1) 284 fd2.write(d2) 285 fd1.close() 286 fd2.close() 287 288 return diffFileData(tmpFile1, tmpFile2) 289 290 ##### 291 # Misc utility functions 292 # 293 294 # Prune off the leading prefix from string s 295 def str_prefix_trunc(s, prefix) : 296 snipLen = len(prefix) 297 return s[snipLen:] 298 299 # 300 # Prune off leading proto path goo (if there is one) to yield 301 # the deliverable's eventual path relative to root 302 # e.g. proto.base/root_sparc/usr/src/cmd/prstat => usr/src/cmd/prstat 303 # 304 def fnFormat(fn) : 305 root_arch_str = "root_" + arch 306 307 pos = fn.find(root_arch_str) 308 if pos == -1 : 309 return fn 310 311 pos = fn.find("/", pos) 312 if pos == -1 : 313 return fn 314 315 return fn[pos + 1:] 316 317 ##### 318 # Usage / argument processing 319 # 320 321 # 322 # Display usage message 323 # 324 def usage() : 325 sys.stdout.flush() 326 print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new 327 -d Print debug messages about the progress 328 -v Do not truncate observed diffs in results 329 -V Log *all* ELF sect diffs vs. logging the first diff found 330 -t Use onbld tools in $SRC/tools 331 -r Log results and observed differences 332 -s Produce sorted list of differences 333 -i Tell wsdiff which objects to compare via an input file list""" 334 sys.exit(1) 335 336 # 337 # Process command line options 338 # 339 def args() : 340 341 global debugon 342 global logging 343 global vdiffs 344 global reportAllSects 345 global sorted 346 347 validOpts = 'di:r:vVst?' 348 349 baseRoot = "" 350 ptchRoot = "" 351 fileNamesFile = "" 352 results = "" 353 localTools = False 354 355 # getopt.getopt() returns: 356 # an option/value tuple 357 # a list of remaining non-option arguments 358 # 359 # A correct wsdiff invocation will have exactly two non option 360 # arguments, the paths to the base (old), ptch (new) proto areas 361 try: 362 optlist, args = getopt.getopt(sys.argv[1:], validOpts) 363 except getopt.error, val: 364 usage() 365 366 if len(args) != 2 : 367 usage(); 368 369 for opt,val in optlist : 370 if opt == '-d' : 371 debugon = True 372 elif opt == '-i' : 373 fileNamesFile = val 374 elif opt == '-r' : 375 results = val 376 logging = True 377 elif opt == '-s' : 378 sorted = True 379 elif opt == '-v' : 380 vdiffs = True 381 elif opt == '-V' : 382 reportAllSects = True 383 elif opt == '-t': 384 localTools = True 385 else: 386 usage() 387 388 baseRoot = args[0] 389 ptchRoot = args[1] 390 391 if len(baseRoot) == 0 or len(ptchRoot) == 0 : 392 usage() 393 394 if logging and len(results) == 0 : 395 usage() 396 397 if vdiffs and not logging : 398 error("The -v option requires a results file (-r)") 399 sys.exit(1) 400 401 if reportAllSects and not logging : 402 error("The -V option requires a results file (-r)") 403 sys.exit(1) 404 405 # alphabetical order 406 return baseRoot, fileNamesFile, localTools, ptchRoot, results 407 408 ##### 409 # File identification 410 # 411 412 # 413 # Identify the file type. 414 # If it's not ELF, use the file extension to identify 415 # certain file types that require special handling to 416 # compare. Otherwise just return a basic "ASCII" type. 417 # 418 def getTheFileType(f) : 419 420 extensions = { 'a' : 'ELF Object Archive', 421 'jar' : 'Java Archive', 422 'html' : 'HTML', 423 'ln' : 'Lint Library', 424 'db' : 'Sqlite Database' } 425 426 try: 427 if os.stat(f)[ST_SIZE] == 0 : 428 return 'ASCII' 429 except: 430 error("failed to stat " + f) 431 return 'Error' 432 433 if isELF(f) == 1 : 434 return 'ELF' 435 436 fnamelist = f.split('.') 437 if len(fnamelist) > 1 : # Test the file extension 438 extension = fnamelist[-1] 439 if extension in extensions.keys(): 440 return extensions[extension] 441 442 return 'ASCII' 443 444 # 445 # Return non-zero if "f" is an ELF file 446 # 447 elfmagic = '\177ELF' 448 def isELF(f) : 449 try: 450 fd = open(f) 451 except: 452 error("failed to open: " + f) 453 return 0 454 magic = fd.read(len(elfmagic)) 455 fd.close() 456 457 if magic == elfmagic : 458 return 1 459 return 0 460 461 # 462 # Return non-zero is "f" is binary. 463 # Consider the file to be binary if it contains any null characters 464 # 465 def isBinary(f) : 466 try: 467 fd = open(f) 468 except: 469 error("failed to open: " + f) 470 return 0 471 s = fd.read() 472 fd.close() 473 474 if s.find('\0') == -1 : 475 return 0 476 else : 477 return 1 478 479 ##### 480 # Directory traversal and file finding 481 # 482 483 # 484 # Return a sorted list of files found under the specified directory 485 # 486 def findFiles(d) : 487 for path, subdirs, files in os.walk(d) : 488 files.sort() 489 for name in files : 490 yield os.path.join(path, name) 491 492 # 493 # Examine all files in base, ptch 494 # 495 # Return a list of files appearing in both proto areas, 496 # a list of new files (files found only in ptch) and 497 # a list of deleted files (files found only in base) 498 # 499 def protoCatalog(base, ptch) : 500 501 compFiles = [] # List of files in both proto areas 502 ptchList = [] # List of file in patch proto area 503 504 newFiles = [] # New files detected 505 deletedFiles = [] # Deleted files 506 507 debug("Getting the list of files in the base area"); 508 baseFilesList = list(findFiles(base)) 509 baseStringLength = len(base) 510 debug("Found " + str(len(baseFilesList)) + " files") 511 512 debug("Getting the list of files in the patch area"); 513 ptchFilesList = list(findFiles(ptch)) 514 ptchStringLength = len(ptch) 515 debug("Found " + str(len(ptchFilesList)) + " files") 516 517 # Inventory files in the base proto area 518 debug("Determining the list of regular files in the base area"); 519 for fn in baseFilesList : 520 if os.path.islink(fn) : 521 continue 522 523 fileName = fn[baseStringLength:] 524 compFiles.append(fileName) 525 debug("Found " + str(len(compFiles)) + " files") 526 527 # Inventory files in the patch proto area 528 debug("Determining the list of regular files in the patch area"); 529 for fn in ptchFilesList : 530 if os.path.islink(fn) : 531 continue 532 533 fileName = fn[ptchStringLength:] 534 ptchList.append(fileName) 535 debug("Found " + str(len(ptchList)) + " files") 536 537 # Deleted files appear in the base area, but not the patch area 538 debug("Searching for deleted files by comparing the lists") 539 for fileName in compFiles : 540 if not fileName in ptchList : 541 deletedFiles.append(fileName) 542 debug("Found " + str(len(deletedFiles)) + " deleted files") 543 544 # Eliminate "deleted" files from the list of objects appearing 545 # in both the base and patch proto areas 546 debug("Eliminating deleted files from the list of objects") 547 for fileName in deletedFiles : 548 try: 549 compFiles.remove(fileName) 550 except: 551 error("filelist.remove() failed") 552 debug("List for comparison reduced to " + str(len(compFiles)) \ 553 + " files") 554 555 # New files appear in the patch area, but not the base 556 debug("Getting the list of newly added files") 557 for fileName in ptchList : 558 if not fileName in compFiles : 559 newFiles.append(fileName) 560 debug("Found " + str(len(newFiles)) + " new files") 561 562 return compFiles, newFiles, deletedFiles 563 564 # 565 # Examine the files listed in the input file list 566 # 567 # Return a list of files appearing in both proto areas, 568 # a list of new files (files found only in ptch) and 569 # a list of deleted files (files found only in base) 570 # 571 def flistCatalog(base, ptch, flist) : 572 compFiles = [] # List of files in both proto areas 573 newFiles = [] # New files detected 574 deletedFiles = [] # Deleted files 575 576 try: 577 fd = open(flist, "r") 578 except: 579 error("could not open: " + flist) 580 cleanup(1) 581 582 files = [] 583 files = fd.readlines() 584 fd.close() 585 586 for f in files : 587 ptch_present = True 588 base_present = True 589 590 if f == '\n' : 591 continue 592 593 # the fileNames have a trailing '\n' 594 f = f.rstrip() 595 596 # The objects in the file list have paths relative 597 # to $ROOT or to the base/ptch directory specified on 598 # the command line. 599 # If it's relative to $ROOT, we'll need to add back the 600 # root_`uname -p` goo we stripped off in fnFormat() 601 if os.path.exists(base + f) : 602 fn = f; 603 elif os.path.exists(base + "root_" + arch + "/" + f) : 604 fn = "root_" + arch + "/" + f 605 else : 606 base_present = False 607 608 if base_present : 609 if not os.path.exists(ptch + fn) : 610 ptch_present = False 611 else : 612 if os.path.exists(ptch + f) : 613 fn = f 614 elif os.path.exists(ptch + "root_" + arch + "/" + f) : 615 fn = "root_" + arch + "/" + f 616 else : 617 ptch_present = False 618 619 if os.path.islink(base + fn) : # ignore links 620 base_present = False 621 if os.path.islink(ptch + fn) : 622 ptch_present = False 623 624 if base_present and ptch_present : 625 compFiles.append(fn) 626 elif base_present : 627 deletedFiles.append(fn) 628 elif ptch_present : 629 newFiles.append(fn) 630 else : 631 if os.path.islink(base + fn) and \ 632 os.path.islink(ptch + fn) : 633 continue 634 error(f + " in file list, but not in either tree. " + \ 635 "Skipping...") 636 637 return compFiles, newFiles, deletedFiles 638 639 640 # 641 # Build a fully qualified path to an external tool/utility. 642 # Consider the default system locations. For onbld tools, if 643 # the -t option was specified, we'll try to use built tools in $SRC tools, 644 # and otherwise, we'll fall back on /opt/onbld/ 645 # 646 def find_tool(tool) : 647 648 # First, check what was passed 649 if os.path.exists(tool) : 650 return tool 651 652 # Next try in wsdiff path 653 for pdir in wsdiff_path : 654 location = pdir + "/" + tool 655 if os.path.exists(location) : 656 return location + " " 657 658 location = pdir + "/" + arch + "/" + tool 659 if os.path.exists(location) : 660 return location + " " 661 662 error("Could not find path to: " + tool); 663 sys.exit(1); 664 665 666 ##### 667 # ELF file comparison helper routines 668 # 669 670 # 671 # Return a dictionary of ELF section types keyed by section name 672 # 673 def get_elfheader(f) : 674 675 header = {} 676 677 hstring = commands.getoutput(elfdump_cmd + " -c " + f) 678 679 if len(hstring) == 0 : 680 error("Failed to dump ELF header for " + f) 681 raise 682 return 683 684 # elfdump(1) dumps the section headers with the section name 685 # following "sh_name:", and the section type following "sh_type:" 686 sections = hstring.split("Section Header") 687 for sect in sections : 688 datap = sect.find("sh_name:"); 689 if datap == -1 : 690 continue 691 section = sect[datap:].split()[1] 692 datap = sect.find("sh_type:"); 693 if datap == -1 : 694 error("Could not get type for sect: " + section + \ 695 " in " + f) 696 sh_type = sect[datap:].split()[2] 697 header[section] = sh_type 698 699 return header 700 701 # 702 # Extract data in the specified ELF section from the given file 703 # 704 def extract_elf_section(f, section) : 705 706 data = commands.getoutput(dump_cmd + " -sn " + section + " " + f) 707 708 if len(data) == 0 : 709 error(dump_cmd + "yielded no data on section " + section + \ 710 " of " + f) 711 raise 712 return 713 714 # dump(1) displays the file name to start... 715 # get past it to the data itself 716 dbegin = data.find(":") + 1 717 data = data[dbegin:]; 718 719 return (data) 720 721 # 722 # Return a (hopefully meaningful) human readable set of diffs 723 # for the specified ELF section between f1 and f2 724 # 725 # Depending on the section, various means for dumping and diffing 726 # the data may be employed. 727 # 728 text_sections = [ '.text', '.init', '.fini' ] 729 def diff_elf_section(f1, f2, section, sh_type) : 730 731 t = threading.currentThread() 732 tmpFile1 = tmpDir1 + os.path.basename(f1) + t.getName() 733 tmpFile2 = tmpDir2 + os.path.basename(f2) + t.getName() 734 735 if (sh_type == "SHT_RELA") : # sh_type == SHT_RELA 736 cmd1 = elfdump_cmd + " -r " + f1 + " > " + tmpFile1 737 cmd2 = elfdump_cmd + " -r " + f2 + " > " + tmpFile2 738 elif (section == ".group") : 739 cmd1 = elfdump_cmd + " -g " + f1 + " > " + tmpFile1 740 cmd2 = elfdump_cmd + " -g " + f2 + " > " + tmpFile2 741 elif (section == ".hash") : 742 cmd1 = elfdump_cmd + " -h " + f1 + " > " + tmpFile1 743 cmd2 = elfdump_cmd + " -h " + f2 + " > " + tmpFile2 744 elif (section == ".dynamic") : 745 cmd1 = elfdump_cmd + " -d " + f1 + " > " + tmpFile1 746 cmd2 = elfdump_cmd + " -d " + f2 + " > " + tmpFile2 747 elif (section == ".got") : 748 cmd1 = elfdump_cmd + " -G " + f1 + " > " + tmpFile1 749 cmd2 = elfdump_cmd + " -G " + f2 + " > " + tmpFile2 750 elif (section == ".SUNW_cap") : 751 cmd1 = elfdump_cmd + " -H " + f1 + " > " + tmpFile1 752 cmd2 = elfdump_cmd + " -H " + f2 + " > " + tmpFile2 753 elif (section == ".interp") : 754 cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1 755 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2 756 elif (section == ".symtab" or section == ".dynsym") : 757 cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \ 758 " > " + tmpFile1 759 cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \ 760 " > " + tmpFile2 761 elif (section in text_sections) : 762 # dis sometimes complains when it hits something it doesn't 763 # know how to disassemble. Just ignore it, as the output 764 # being generated here is human readable, and we've already 765 # correctly flagged the difference. 766 cmd1 = dis_cmd + " -t " + section + " " + f1 + \ 767 " 2>/dev/null | grep -v disassembly > " + tmpFile1 768 cmd2 = dis_cmd + " -t " + section + " " + f2 + \ 769 " 2>/dev/null | grep -v disassembly > " + tmpFile2 770 else : 771 cmd1 = elfdump_cmd + " -w " + tmpFile1 + " -N " + \ 772 section + " " + f1 773 cmd2 = elfdump_cmd + " -w " + tmpFile2 + " -N " + \ 774 section + " " + f2 775 776 os.system(cmd1) 777 os.system(cmd2) 778 779 data = diffFileData(tmpFile1, tmpFile2) 780 781 # remove temp files as we no longer need them 782 try: 783 os.unlink(tmpFile1) 784 except OSError, e: 785 error("diff_elf_section: unlink failed %s" % e) 786 try: 787 os.unlink(tmpFile2) 788 except OSError, e: 789 error("diff_elf_section: unlink failed %s" % e) 790 791 return (data) 792 793 # 794 # compare the relevant sections of two ELF binaries 795 # and report any differences 796 # 797 # Returns: 1 if any differenes found 798 # 0 if no differences found 799 # -1 on error 800 # 801 802 # Sections deliberately not considered when comparing two ELF 803 # binaries. Differences observed in these sections are not considered 804 # significant where patch deliverable identification is concerned. 805 sections_to_skip = [ ".SUNW_signature", 806 ".comment", 807 ".SUNW_ctf", 808 ".debug", 809 ".plt", 810 ".rela.bss", 811 ".rela.plt", 812 ".line", 813 ".note", 814 ".compcom", 815 ] 816 817 sections_preferred = [ ".rodata.str1.8", 818 ".rodata.str1.1", 819 ".rodata", 820 ".data1", 821 ".data", 822 ".text", 823 ] 824 825 def compareElfs(base, ptch, quiet) : 826 827 global logging 828 829 try: 830 base_header = get_elfheader(base) 831 except: 832 return 833 sections = base_header.keys() 834 835 try: 836 ptch_header = get_elfheader(ptch) 837 except: 838 return 839 e2_only_sections = ptch_header.keys() 840 841 e1_only_sections = [] 842 843 fileName = fnFormat(base) 844 845 # Derive the list of ELF sections found only in 846 # either e1 or e2. 847 for sect in sections : 848 if not sect in e2_only_sections : 849 e1_only_sections.append(sect) 850 else : 851 e2_only_sections.remove(sect) 852 853 if len(e1_only_sections) > 0 : 854 if quiet : 855 return 1 856 857 data = "" 858 if logging : 859 slist = "" 860 for sect in e1_only_sections : 861 slist = slist + sect + "\t" 862 data = "ELF sections found in " + \ 863 base + " but not in " + ptch + \ 864 "\n\n" + slist 865 866 difference(fileName, "ELF", data) 867 return 1 868 869 if len(e2_only_sections) > 0 : 870 if quiet : 871 return 1 872 873 data = "" 874 if logging : 875 slist = "" 876 for sect in e2_only_sections : 877 slist = slist + sect + "\t" 878 data = "ELF sections found in " + \ 879 ptch + " but not in " + base + \ 880 "\n\n" + slist 881 882 difference(fileName, "ELF", data) 883 return 1 884 885 # Look for preferred sections, and put those at the 886 # top of the list of sections to compare 887 for psect in sections_preferred : 888 if psect in sections : 889 sections.remove(psect) 890 sections.insert(0, psect) 891 892 # Compare ELF sections 893 first_section = True 894 for sect in sections : 895 896 if sect in sections_to_skip : 897 continue 898 899 try: 900 s1 = extract_elf_section(base, sect); 901 except: 902 return 903 904 try: 905 s2 = extract_elf_section(ptch, sect); 906 except: 907 return 908 909 if len(s1) != len (s2) or s1 != s2: 910 if not quiet: 911 sh_type = base_header[sect] 912 data = diff_elf_section(base, ptch, \ 913 sect, sh_type) 914 915 # If all ELF sections are being reported, then 916 # invoke difference() to flag the file name to 917 # stdout only once. Any other section differences 918 # should be logged to the results file directly 919 if not first_section : 920 log_difference(fileName, \ 921 "ELF " + sect, data) 922 else : 923 difference(fileName, "ELF " + sect, \ 924 data) 925 926 if not reportAllSects : 927 return 1 928 first_section = False 929 930 return 0 931 932 ##### 933 # recursively remove 2 directories 934 # 935 # Used for removal of temporary directory strucures (ignores any errors). 936 # 937 def clearTmpDirs(dir1, dir2) : 938 939 if os.path.isdir(dir1) > 0 : 940 shutil.rmtree(dir1, True) 941 942 if os.path.isdir(dir2) > 0 : 943 shutil.rmtree(dir2, True) 944 945 946 ##### 947 # Archive object comparison 948 # 949 # Returns 1 if difference detected 950 # 0 if no difference detected 951 # -1 on error 952 # 953 def compareArchives(base, ptch, fileType) : 954 955 fileName = fnFormat(base) 956 t = threading.currentThread() 957 ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() 958 ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName() 959 960 # 961 # Be optimistic and first try a straight file compare 962 # as it will allow us to finish up quickly. 963 # 964 if compareBasic(base, ptch, True, fileType) == 0 : 965 return 0 966 967 try: 968 os.makedirs(ArchTmpDir1) 969 except OSError, e: 970 error("compareArchives: makedir failed %s" % e) 971 return -1 972 try: 973 os.makedirs(ArchTmpDir2) 974 except OSError, e: 975 error("compareArchives: makedir failed %s" % e) 976 return -1 977 978 # copy over the objects to the temp areas, and 979 # unpack them 980 baseCmd = "cp -fp " + base + " " + ArchTmpDir1 981 status, output = commands.getstatusoutput(baseCmd) 982 if status != 0 : 983 error(baseCmd + " failed: " + output) 984 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 985 return -1 986 987 ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2 988 status, output = commands.getstatusoutput(ptchCmd) 989 if status != 0 : 990 error(ptchCmd + " failed: " + output) 991 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 992 return -1 993 994 bname = string.split(fileName, '/')[-1] 995 if fileType == "Java Archive" : 996 baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \ 997 "; rm -f " + bname + " META-INF/MANIFEST.MF" 998 ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \ 999 "; rm -f " + bname + " META-INF/MANIFEST.MF" 1000 elif fileType == "ELF Object Archive" : 1001 baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \ 1002 bname + "; rm -f " + bname 1003 ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \ 1004 bname + "; rm -f " + bname 1005 else : 1006 error("unexpected file type: " + fileType) 1007 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1008 return -1 1009 1010 os.system(baseCmd) 1011 os.system(ptchCmd) 1012 1013 baseFlist = list(findFiles(ArchTmpDir1)) 1014 ptchFlist = list(findFiles(ArchTmpDir2)) 1015 1016 # Trim leading path off base/ptch file lists 1017 flist = [] 1018 for fn in baseFlist : 1019 flist.append(str_prefix_trunc(fn, ArchTmpDir1)) 1020 baseFlist = flist 1021 1022 flist = [] 1023 for fn in ptchFlist : 1024 flist.append(str_prefix_trunc(fn, ArchTmpDir2)) 1025 ptchFlist = flist 1026 1027 for fn in ptchFlist : 1028 if not fn in baseFlist : 1029 difference(fileName, fileType, \ 1030 fn + " added to " + fileName) 1031 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1032 return 1 1033 1034 for fn in baseFlist : 1035 if not fn in ptchFlist : 1036 difference(fileName, fileType, \ 1037 fn + " removed from " + fileName) 1038 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1039 return 1 1040 1041 differs = compareOneFile((ArchTmpDir1 + fn), \ 1042 (ArchTmpDir2 + fn), True) 1043 if differs : 1044 difference(fileName, fileType, \ 1045 fn + " in " + fileName + " differs") 1046 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1047 return 1 1048 1049 clearTmpDirs(ArchTmpDir1, ArchTmpDir2) 1050 return 0 1051 1052 ##### 1053 # (Basic) file comparison 1054 # 1055 # There's some special case code here for Javadoc HTML files 1056 # 1057 # Returns 1 if difference detected 1058 # 0 if no difference detected 1059 # -1 on error 1060 # 1061 def compareBasic(base, ptch, quiet, fileType) : 1062 1063 fileName = fnFormat(base); 1064 1065 if quiet and os.stat(base)[ST_SIZE] != os.stat(ptch)[ST_SIZE] : 1066 return 1 1067 1068 try: 1069 baseFile = open(base) 1070 except: 1071 error("could not open " + base) 1072 return -1 1073 try: 1074 ptchFile = open(ptch) 1075 except: 1076 error("could not open " + ptch) 1077 return -1 1078 1079 baseData = baseFile.read() 1080 ptchData = ptchFile.read() 1081 1082 baseFile.close() 1083 ptchFile.close() 1084 1085 needToSnip = False 1086 if fileType == "HTML" : 1087 needToSnip = True 1088 toSnipBeginStr = "<!-- Generated by javadoc" 1089 toSnipEndStr = "-->\n" 1090 1091 if needToSnip : 1092 toSnipBegin = string.find(baseData, toSnipBeginStr) 1093 if toSnipBegin != -1 : 1094 toSnipEnd = string.find(baseData[toSnipBegin:], \ 1095 toSnipEndStr) + \ 1096 len(toSnipEndStr) 1097 baseData = baseData[:toSnipBegin] + \ 1098 baseData[toSnipBegin + toSnipEnd:] 1099 ptchData = ptchData[:toSnipBegin] + \ 1100 ptchData[toSnipBegin + toSnipEnd:] 1101 1102 if quiet : 1103 if baseData != ptchData : 1104 return 1 1105 else : 1106 if len(baseData) != len(ptchData) or baseData != ptchData : 1107 diffs = diffData(base, ptch, baseData, ptchData) 1108 difference(fileName, fileType, diffs) 1109 return 1 1110 return 0 1111 1112 1113 ##### 1114 # Compare two objects by producing a data dump from 1115 # each object, and then comparing the dump data 1116 # 1117 # Returns: 1 if a difference is detected 1118 # 0 if no difference detected 1119 # -1 upon error 1120 # 1121 def compareByDumping(base, ptch, quiet, fileType) : 1122 1123 fileName = fnFormat(base); 1124 t = threading.currentThread() 1125 tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName() 1126 tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() 1127 1128 if fileType == "Lint Library" : 1129 baseCmd = lintdump_cmd + " -ir " + base + \ 1130 " | egrep -v '(LINTOBJ|LINTMOD):'" + \ 1131 " | grep -v PASS[1-3]:" + \ 1132 " > " + tmpFile1 1133 ptchCmd = lintdump_cmd + " -ir " + ptch + \ 1134 " | egrep -v '(LINTOBJ|LINTMOD):'" + \ 1135 " | grep -v PASS[1-3]:" + \ 1136 " > " + tmpFile2 1137 elif fileType == "Sqlite Database" : 1138 baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \ 1139 tmpFile1 1140 ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \ 1141 tmpFile2 1142 1143 os.system(baseCmd) 1144 os.system(ptchCmd) 1145 1146 try: 1147 baseFile = open(tmpFile1) 1148 except: 1149 error("could not open: " + tmpFile1) 1150 return 1151 try: 1152 ptchFile = open(tmpFile2) 1153 except: 1154 error("could not open: " + tmpFile2) 1155 return 1156 1157 baseData = baseFile.read() 1158 ptchData = ptchFile.read() 1159 1160 baseFile.close() 1161 ptchFile.close() 1162 1163 if len(baseData) != len(ptchData) or baseData != ptchData : 1164 if not quiet : 1165 data = diffFileData(tmpFile1, tmpFile2); 1166 try: 1167 os.unlink(tmpFile1) 1168 except OSError, e: 1169 error("compareByDumping: unlink failed %s" % e) 1170 try: 1171 os.unlink(tmpFile2) 1172 except OSError, e: 1173 error("compareByDumping: unlink failed %s" % e) 1174 difference(fileName, fileType, data) 1175 return 1 1176 1177 # Remove the temporary files now. 1178 try: 1179 os.unlink(tmpFile1) 1180 except OSError, e: 1181 error("compareByDumping: unlink failed %s" % e) 1182 try: 1183 os.unlink(tmpFile2) 1184 except OSError, e: 1185 error("compareByDumping: unlink failed %s" % e) 1186 1187 return 0 1188 1189 ##### 1190 # 1191 # SIGINT signal handler. Changes thread control variable to tell the threads 1192 # to finish their current job and exit. 1193 # 1194 def discontinue_processing(signl, frme): 1195 global keep_processing 1196 1197 print >> sys.stderr, "Caught Ctrl-C, stopping the threads" 1198 keep_processing = False 1199 1200 return 0 1201 1202 ##### 1203 # 1204 # worker thread for changedFiles processing 1205 # 1206 class workerThread(threading.Thread) : 1207 def run(self): 1208 global wset_lock 1209 global changedFiles 1210 global baseRoot 1211 global ptchRoot 1212 global keep_processing 1213 1214 while (keep_processing) : 1215 # grab the lock to changedFiles and remove one member 1216 # and process it 1217 wset_lock.acquire() 1218 try : 1219 fn = changedFiles.pop() 1220 except IndexError : 1221 # there is nothing more to do 1222 wset_lock.release() 1223 return 1224 wset_lock.release() 1225 1226 base = baseRoot + fn 1227 ptch = ptchRoot + fn 1228 1229 compareOneFile(base, ptch, False) 1230 1231 1232 ##### 1233 # Compare two objects. Detect type changes. 1234 # Vector off to the appropriate type specific 1235 # compare routine based on the type. 1236 # 1237 def compareOneFile(base, ptch, quiet) : 1238 1239 # Verify the file types. 1240 # If they are different, indicate this and move on 1241 btype = getTheFileType(base) 1242 ptype = getTheFileType(ptch) 1243 1244 if btype == 'Error' or ptype == 'Error' : 1245 return -1 1246 1247 fileName = fnFormat(base) 1248 1249 if (btype != ptype) : 1250 if not quiet : 1251 difference(fileName, "file type", btype + " to " + ptype) 1252 return 1 1253 else : 1254 fileType = btype 1255 1256 if (fileType == 'ELF') : 1257 return compareElfs(base, ptch, quiet) 1258 1259 elif (fileType == 'Java Archive' or fileType == 'ELF Object Archive') : 1260 return compareArchives(base, ptch, fileType) 1261 1262 elif (fileType == 'HTML') : 1263 return compareBasic(base, ptch, quiet, fileType) 1264 1265 elif ( fileType == 'Lint Library' ) : 1266 return compareByDumping(base, ptch, quiet, fileType) 1267 1268 elif ( fileType == 'Sqlite Database' ) : 1269 return compareByDumping(base, ptch, quiet, fileType) 1270 1271 else : 1272 # it has to be some variety of text file 1273 return compareBasic(base, ptch, quiet, fileType) 1274 1275 # Cleanup and self-terminate 1276 def cleanup(ret) : 1277 1278 debug("Performing cleanup (" + str(ret) + ")") 1279 if os.path.isdir(tmpDir1) > 0 : 1280 shutil.rmtree(tmpDir1) 1281 1282 if os.path.isdir(tmpDir2) > 0 : 1283 shutil.rmtree(tmpDir2) 1284 1285 if logging : 1286 log.close() 1287 1288 sys.exit(ret) 1289 1290 def main() : 1291 1292 # Log file handle 1293 global log 1294 1295 # Globals relating to command line options 1296 global logging, vdiffs, reportAllSects 1297 1298 # Named temporary files / directories 1299 global tmpDir1, tmpDir2 1300 1301 # Command paths 1302 global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, diff_cmd, sqlite_cmd 1303 1304 # Default search path 1305 global wsdiff_path 1306 1307 # Essentially "uname -p" 1308 global arch 1309 1310 # changed files for worker thread processing 1311 global changedFiles 1312 global baseRoot 1313 global ptchRoot 1314 1315 # Sort the list of files from a temporary file 1316 global sorted 1317 global differentFiles 1318 1319 # Debugging indicator 1320 global debugon 1321 1322 # Some globals need to be initialized 1323 debugon = logging = vdiffs = reportAllSects = sorted = False 1324 1325 1326 # Process command line arguments 1327 # Return values are returned from args() in alpha order 1328 # (Yes, python functions can return multiple values (ewww)) 1329 # Note that args() also set the globals: 1330 # logging to True if verbose logging (to a file) was enabled 1331 # vdiffs to True if logged differences aren't to be truncated 1332 # reportAllSects to True if all ELF section differences are to be reported 1333 # 1334 baseRoot, fileNamesFile, localTools, ptchRoot, results = args() 1335 1336 # 1337 # Set up the results/log file 1338 # 1339 if logging : 1340 try: 1341 log = open(results, "w") 1342 except: 1343 logging = False 1344 error("failed to open log file: " + log) 1345 sys.exit(1) 1346 1347 dateTimeStr= "# %04d-%02d-%02d at %02d:%02d:%02d" % time.localtime()[:6] 1348 v_info("# This file was produced by wsdiff") 1349 v_info(dateTimeStr) 1350 1351 # Changed files (used only for the sorted case) 1352 if sorted : 1353 differentFiles = [] 1354 1355 # 1356 # Build paths to the tools required tools 1357 # 1358 # Try to look for tools in $SRC/tools if the "-t" option 1359 # was specified 1360 # 1361 arch = commands.getoutput("uname -p") 1362 if localTools : 1363 try: 1364 src = os.environ['SRC'] 1365 except: 1366 error("-t specified, but $SRC not set. Cannot find $SRC/tools") 1367 src = "" 1368 if len(src) > 0 : 1369 wsdiff_path.insert(0, src + "/tools/proto/opt/onbld/bin") 1370 1371 lintdump_cmd = find_tool("lintdump") 1372 elfdump_cmd = find_tool("elfdump") 1373 dump_cmd = find_tool("dump") 1374 od_cmd = find_tool("od") 1375 dis_cmd = find_tool("dis") 1376 diff_cmd = find_tool("diff") 1377 sqlite_cmd = find_tool("sqlite") 1378 1379 # 1380 # Set resource limit for number of open files as high as possible. 1381 # This might get handy with big number of threads. 1382 # 1383 (nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE) 1384 try: 1385 resource.setrlimit(resource.RLIMIT_NOFILE, 1386 (nofile_hard, nofile_hard)) 1387 except: 1388 error("cannot set resource limits for number of open files") 1389 sys.exit(1) 1390 1391 # 1392 # validate the base and patch paths 1393 # 1394 if baseRoot[-1] != '/' : 1395 baseRoot += '/' 1396 1397 if ptchRoot[-1] != '/' : 1398 ptchRoot += '/' 1399 1400 if not os.path.exists(baseRoot) : 1401 error("old proto area: " + baseRoot + " does not exist") 1402 sys.exit(1) 1403 1404 if not os.path.exists(ptchRoot) : 1405 error("new proto area: " + ptchRoot + \ 1406 " does not exist") 1407 sys.exit(1) 1408 1409 # 1410 # log some information identifying the run 1411 # 1412 v_info("Old proto area: " + baseRoot) 1413 v_info("New proto area: " + ptchRoot) 1414 v_info("Results file: " + results + "\n") 1415 1416 # 1417 # Set up the temporary directories / files 1418 # Could use python's tmpdir routines, but these should 1419 # be easier to identify / keep around for debugging 1420 pid = os.getpid() 1421 tmpDir1 = "/tmp/wsdiff_tmp1_" + str(pid) + "/" 1422 tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/" 1423 try: 1424 os.makedirs(tmpDir1) 1425 except OSError, e: 1426 error("main: makedir failed %s" % e) 1427 try: 1428 os.makedirs(tmpDir2) 1429 except OSError, e: 1430 error("main: makedir failed %s" % e) 1431 1432 # Derive a catalog of new, deleted, and to-be-compared objects 1433 # either from the specified base and patch proto areas, or from 1434 # from an input file list 1435 newOrDeleted = False 1436 1437 if fileNamesFile != "" : 1438 changedFiles, newFiles, deletedFiles = \ 1439 flistCatalog(baseRoot, ptchRoot, fileNamesFile) 1440 else : 1441 changedFiles, newFiles, deletedFiles = \ 1442 protoCatalog(baseRoot, ptchRoot) 1443 1444 if len(newFiles) > 0 : 1445 newOrDeleted = True 1446 info("\nNew objects found: ") 1447 1448 if sorted : 1449 newFiles.sort() 1450 for fn in newFiles : 1451 info(fnFormat(fn)) 1452 1453 if len(deletedFiles) > 0 : 1454 newOrDeleted = True 1455 info("\nObjects removed: ") 1456 1457 if sorted : 1458 deletedFiles.sort() 1459 for fn in deletedFiles : 1460 info(fnFormat(fn)) 1461 1462 if newOrDeleted : 1463 info("\nChanged objects: ") 1464 if sorted : 1465 debug("The list will appear after the processing is done") 1466 1467 # Here's where all the heavy lifting happens 1468 # Perform a comparison on each object appearing in 1469 # both proto areas. compareOneFile will examine the 1470 # file types of each object, and will vector off to 1471 # the appropriate comparison routine, where the compare 1472 # will happen, and any differences will be reported / logged 1473 1474 # determine maximum number of worker threads by using 1475 # DMAKE_MAX_JOBS environment variable set by nightly(1) 1476 # or get number of CPUs in the system 1477 try: 1478 max_threads = int(os.environ['DMAKE_MAX_JOBS']) 1479 except: 1480 max_threads = os.sysconf("SC_NPROCESSORS_ONLN") 1481 # If we cannot get number of online CPUs in the system 1482 # run unparallelized otherwise bump the number up 20% 1483 # to achieve best results. 1484 if max_threads == -1 : 1485 max_threads = 1 1486 else : 1487 max_threads += max_threads/5 1488 1489 # Set signal handler to attempt graceful exit 1490 debug("Setting signal handler") 1491 signal.signal( signal.SIGINT, discontinue_processing ) 1492 1493 # Create and unleash the threads 1494 # Only at most max_threads must be running at any moment 1495 mythreads = [] 1496 debug("Spawning " + str(max_threads) + " threads"); 1497 for i in range(max_threads) : 1498 thread = workerThread() 1499 mythreads.append(thread) 1500 mythreads[i].start() 1501 1502 # Wait for the threads to finish and do cleanup if interrupted 1503 debug("Waiting for the threads to finish") 1504 while True: 1505 if not True in [thread.isAlive() for thread in mythreads]: 1506 break 1507 else: 1508 # Some threads are still going 1509 time.sleep(1) 1510 1511 # Interrupted by SIGINT 1512 if keep_processing == False : 1513 cleanup(1) 1514 1515 # If the list of differences was sorted it is stored in an array 1516 if sorted : 1517 differentFiles.sort() 1518 for f in differentFiles : 1519 info(fnFormat(f)) 1520 1521 # We're done, cleanup. 1522 cleanup(0) 1523 1524 if __name__ == '__main__' : 1525 try: 1526 main() 1527 except KeyboardInterrupt : 1528 cleanup(1); 1529 1530 1531