xref: /illumos-gate/usr/src/tools/scripts/wsdiff.py (revision b0d58672df8644288a5fdce6ae229eaaa0db2de7)
1#!@TOOLS_PYTHON@ -Es
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
24#
25
26#
27# wsdiff(1) is a tool that can be used to determine which compiled objects
28# have changed as a result of a given source change. Developers backporting
29# new features, RFEs and bug fixes need to be able to identify the set of
30# patch deliverables necessary for feature/fix realization on a patched system.
31#
32# The tool works by comparing objects in two trees/proto areas (one build with,
33# and without the source changes.
34#
35# Using wsdiff(1) is fairly simple:
36#	- Bringover to a fresh workspace
37#	- Perform a full non-debug build (clobber if workspace isn't fresh)
38#	- Move the proto area aside, call it proto.old, or something.
39#	- Integrate your changes to the workspace
40#	- Perform another full non-debug clobber build.
41#	- Use wsdiff(1) to see what changed:
42#		$ wsdiff proto.old proto
43#
44# By default, wsdiff will print the list of changed objects / deliverables to
45# stdout. If a results file is specified via -r, the list of differing objects,
46# and details about why wsdiff(1) thinks they are different will be logged to
47# the results file.
48#
49# By invoking nightly(1) with the -w option to NIGHTLY_FLAGS, nightly(1) will
50# use wsdiff(1) to report on what objects changed since the last build.
51#
52# For patch deliverable purposes, it's advised to have nightly do a clobber,
53# non-debug build.
54#
55# Think about the results. Was something flagged that you don't expect? Go look
56# at the results file to see details about the differences.
57#
58# Use the -i option in conjunction with -v and -V to dive deeper and have
59# wsdiff(1) report with more verbosity.
60#
61# Usage: wsdiff [-dstuvV] [-U lines] [-r results ] [-i filelist ] old new
62#
63# Where "old" is the path to the proto area build without the changes, and
64# "new" is the path to the proto area built with the changes. The following
65# options are supported:
66#
67#       -d      Print debug messages about the progress
68#       -i      Tell wsdiff which objects to compare via an input file list
69#       -r      Log results and observed differences
70#       -s      Produce sorted list of differences
71#       -t      Use onbld tools in $SRC/tools
72#       -u      Produce unified diff output
73#       -U      Produce unified diff output with <lines> lines of context
74#       -v      Do not truncate observed diffs in results
75#       -V      Log *all* ELF sect diffs vs. logging the first diff found
76
77from __future__ import print_function
78import datetime, fnmatch, getopt, os, profile, io, subprocess
79import re, resource, select, shutil, signal, string, struct, sys, tempfile
80import time, threading
81from stat import *
82
83PY3 = sys.version_info[0] == 3
84
85# Human readable diffs truncated by default if longer than this
86# Specifying -v on the command line will override
87diffs_sz_thresh = 4096
88diff_args = ''
89
90# Lock name	 Provides exclusive access to
91# --------------+------------------------------------------------
92# output_lock	 standard output or temporary file (difference())
93# log_lock	 the results file (log_difference())
94# wset_lock	 changedFiles list (workerThread())
95output_lock = threading.Lock()
96log_lock = threading.Lock()
97wset_lock = threading.Lock()
98
99# Variable for thread control
100keep_processing = True
101
102# Default search path for wsdiff
103wsdiff_path = [ "/usr/bin",
104		"/usr/ccs/bin",
105		"/lib/svc/bin",
106		"/opt/onbld/bin" ]
107
108# These are objects that wsdiff will notice look different, but will not report.
109# Existence of an exceptions list, and adding things here is *dangerous*,
110# and therefore the *only* reasons why anything would be listed here is because
111# the objects do not build deterministically, yet we *cannot* fix this.
112#
113wsdiff_exceptions = [
114]
115
116# Path to genunix, used for finding the proto root, and for CTF diff
117genunix = "kernel/amd64/genunix"
118
119if PY3:
120	def getoutput(cmd):
121		import shlex, tempfile
122		f, fpath = tempfile.mkstemp()
123		status = os.system("{ " + cmd + "; } >" +
124			shlex.quote(fpath) + " 2>&1")
125		returncode = os.WEXITSTATUS(status)
126		with os.fdopen(f, mode="r", errors="ignore") as tfile:
127			output = tfile.read()
128		os.unlink(fpath)
129		if output[-1:] == '\n':
130			output = output[:-1]
131		return returncode, output
132else:
133	import commands
134	getoutput = commands.getstatusoutput
135
136#####
137# Logging routines
138#
139
140# Debug message to be printed to the screen, and the log file
141def debug(msg) :
142
143	# Add prefix to highlight debugging message
144	msg = "## " + msg
145	if debugon :
146		output_lock.acquire()
147		print(msg)
148		sys.stdout.flush()
149		output_lock.release()
150		if logging :
151			log_lock.acquire()
152			print(msg, file=log)
153			log.flush()
154			log_lock.release()
155
156# Informational message to be printed to the screen, and the log file
157def info(msg) :
158
159	output_lock.acquire()
160	print(msg)
161	sys.stdout.flush()
162	output_lock.release()
163	if logging :
164		log_lock.acquire()
165		print(msg, file=log)
166		log.flush()
167		log_lock.release()
168
169# Error message to be printed to the screen, and the log file
170def error(msg) :
171
172	output_lock.acquire()
173	print("ERROR: " + msg, file=sys.stderr)
174	sys.stderr.flush()
175	output_lock.release()
176	if logging :
177		log_lock.acquire()
178		print("ERROR: " + msg, file=log)
179		log.flush()
180		log_lock.release()
181
182# Informational message to be printed only to the log, if there is one.
183def v_info(msg) :
184
185	if logging :
186		log_lock.acquire()
187		print(msg, file=log)
188		log.flush()
189		log_lock.release()
190
191#
192# Flag a detected file difference
193# Display the fileName to stdout, and log the difference
194#
195def difference(f, dtype, diffs) :
196
197	if f in wsdiff_exceptions :
198		return
199
200	output_lock.acquire()
201	if o_sorted :
202		differentFiles.append(f)
203	else:
204		print(f)
205		sys.stdout.flush()
206	output_lock.release()
207
208	log_difference(f, dtype, diffs)
209
210#
211# Do the actual logging of the difference to the results file
212#
213def log_difference(f, dtype, diffs) :
214
215	if logging :
216		log_lock.acquire()
217		print(f, file=log)
218		print("NOTE: " + dtype + " difference detected.", file=log)
219
220		difflen = len(diffs)
221		if difflen > 0 :
222			print('', file=log)
223
224			if not vdiffs and difflen > diffs_sz_thresh :
225				print(diffs[:diffs_sz_thresh], file=log)
226				print("... truncated due to length: " +
227				      "use -v to override ...", file=log)
228			else :
229				print(diffs, file=log)
230			print('\n', file=log)
231		log.flush()
232		log_lock.release()
233
234
235#####
236# diff generating routines
237#
238
239#
240# Return human readable diffs from two files
241#
242def diffFileData(tmpf1, tmpf2) :
243
244	binaries = False
245
246	# Filter the data through od(1) if the data is detected
247	# as being binary
248	if isBinary(tmpf1) or isBinary(tmpf2) :
249		binaries = True
250		tmp_od1 = tmpf1 + ".od"
251		tmp_od2 = tmpf2 + ".od"
252
253		cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1
254		os.system(cmd)
255		cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2
256		os.system(cmd)
257
258		tmpf1 = tmp_od1
259		tmpf2 = tmp_od2
260
261	dcmd = "{} {} {} {}".format(diff_cmd, diff_args, tmpf1, tmpf2)
262	try:
263		rc, data = getoutput(dcmd)
264		if rc == 0:
265			# No differences found
266			data = ''
267		# If producing unified output, strip the first two lines
268		# which just show the temporary file names.
269		if diff_args:
270			data = data.split("\n", 2)[-1]
271
272		# Remove the temp files as we no longer need them.
273		if binaries :
274			try:
275				os.unlink(tmp_od1)
276				os.unlink(tmp_od2)
277			except OSError as e:
278				error("diffFileData: unlink failed {}"
279				    .format(e))
280	except:
281		error("failed to get output of command: " + dcmd)
282
283		# Send exception for the failed command up
284		raise
285		return
286
287	return data
288
289#####
290# Misc utility functions
291#
292
293# Prune off the leading prefix from string s
294def str_prefix_trunc(s, prefix) :
295	snipLen = len(prefix)
296	return s[snipLen:]
297
298#
299# Prune off leading proto path goo (if there is one) to yield
300# the deliverable's eventual path relative to root
301# e.g. proto.base/root_sparc/usr/src/cmd/prstat => usr/src/cmd/prstat
302#
303def fnFormat(fn) :
304	global baseWsRoot, ptchWsRoot
305
306	if (baseWsRoot and
307	    len(os.path.commonprefix([fn, baseWsRoot])) == len(baseWsRoot)):
308		return fn[len(baseWsRoot) + 1:]
309
310	if (ptchWsRoot and
311	    len(os.path.commonprefix([fn, ptchWsRoot])) == len(ptchWsRoot)):
312		return fn[len(ptchWsRoot) + 1:]
313
314	# Fall back to looking for the expected root_<arch>[-nd] string
315
316	pos = fn.find("root_" + arch)
317	if pos == -1 :
318		return fn
319
320	pos = fn.find("/", pos)
321	if pos == -1 :
322		return fn
323
324	return fn[pos + 1:]
325
326#
327# Find the path to a proto root given the name of a file or directory under it
328# e.g. proto.base/root_i386-nd/usr/bin => proto.base/root_i386-nd
329#      root/usr/bin => root
330#
331def protoroot(fn):
332	root = fn.rstrip('/')
333	while len(root) > 1:
334		if os.path.isfile(os.path.join(root, genunix)):
335			return root
336		root = os.path.dirname(root)
337
338	# genunix was not found, try and determine the root by checking for
339	# the expected root_<arch>[-nd] string
340
341	pos = fn.find("root_" + arch)
342	if pos == -1:
343		return None
344
345	pos = fn.find("/", pos)
346	if pos == -1:
347		return fn
348
349	return fn[:pos]
350
351#####
352# Usage / argument processing
353#
354
355#
356# Display usage message
357#
358def usage() :
359	sys.stdout.flush()
360	print("""Usage: wsdiff [-dstuvV] [-U lines] [-r results ] [-i filelist ] old new
361        -d      Print debug messages about the progress
362        -i      Tell wsdiff which objects to compare via an input file list
363        -r      Log results and observed differences
364        -s      Produce sorted list of differences
365        -t      Use onbld tools in $SRC/tools
366        -u      Produce unified diff output
367        -U      Produce unified diff output with <lines> lines of context
368        -v      Do not truncate observed diffs in results
369        -V      Log *all* ELF sect diffs vs. logging the first diff found""",
370	    file=sys.stderr)
371	sys.exit(1)
372
373#
374# Process command line options
375#
376def args() :
377
378	global debugon
379	global logging
380	global vdiffs
381	global reportAllSects
382	global o_sorted
383	global diff_args
384
385	validOpts = 'di:r:uU:vVst?'
386
387	baseRoot = ""
388	ptchRoot = ""
389	fileNamesFile = ""
390	results = ""
391	localTools = False
392
393	# getopt.getopt() returns:
394	#	an option/value tuple
395	#	a list of remaining non-option arguments
396	#
397	# A correct wsdiff invocation will have exactly two non option
398	# arguments, the paths to the base (old), ptch (new) proto areas
399	try:
400		optlist, args = getopt.getopt(sys.argv[1:], validOpts)
401	except getopt.error as val:
402		usage()
403
404	if len(args) != 2 :
405		usage();
406
407	for opt,val in optlist :
408		if opt == '-d' :
409			debugon = True
410		elif opt == '-i' :
411			fileNamesFile = val
412		elif opt == '-r' :
413			results = val
414			logging = True
415		elif opt == '-s' :
416			o_sorted = True
417		elif opt == '-u' :
418			diff_args = '-u'
419		elif opt == '-U' :
420			diff_args = '-U' + str(val)
421		elif opt == '-v' :
422			vdiffs = True
423		elif opt == '-V' :
424			reportAllSects = True
425		elif opt == '-t':
426			localTools = True
427		else:
428			usage()
429
430	baseRoot = args[0]
431	ptchRoot = args[1]
432
433	if len(baseRoot) == 0 or len(ptchRoot) == 0 :
434		usage()
435
436	if logging and len(results) == 0 :
437		usage()
438
439	if vdiffs and not logging :
440		error("The -v option requires a results file (-r)")
441		sys.exit(1)
442
443	if reportAllSects and not logging :
444		error("The -V option requires a results file (-r)")
445		sys.exit(1)
446
447	# alphabetical order
448	return	baseRoot, fileNamesFile, localTools, ptchRoot, results
449
450#####
451# File identification
452#
453
454#
455# Identify the file type.
456# If it's not ELF, use the file extension to identify
457# certain file types that require special handling to
458# compare. Otherwise just return a basic "ASCII" type.
459#
460def getTheFileType(f) :
461
462	extensions = { 'a'	:	'ELF Object Archive',
463		       'jar'	:	'Java Archive',
464		       'html'	:	'HTML',
465		       'ln'	:	'Lint Library',
466		       'db'	:	'Sqlite Database' }
467
468	try:
469		if os.stat(f)[ST_SIZE] == 0 :
470			return 'ASCII'
471	except:
472		error("failed to stat " + f)
473		return 'Error'
474
475	if isELF(f) == 1 :
476		return 'ELF'
477
478	fnamelist = f.split('.')
479	if len(fnamelist) > 1 :	# Test the file extension
480		extension = fnamelist[-1]
481		if extension in extensions.keys():
482			return extensions[extension]
483
484	return 'ASCII'
485
486#
487# Return non-zero if "f" is an ELF file
488#
489elfmagic = b'\177ELF'
490def isELF(f) :
491	try:
492		with open(f, mode='rb') as fd:
493			magic = fd.read(len(elfmagic))
494
495		if magic == elfmagic :
496			return 1
497	except:
498		pass
499	return 0
500
501#
502# Return non-zero is "f" is binary.
503# Consider the file to be binary if it contains any null characters
504#
505def isBinary(f) :
506	try:
507		with open(f, mode='rb') as fd:
508			s = fd.read()
509
510		if s.find(b'\0') == -1 :
511			return 0
512	except:
513		pass
514	return 1
515
516#####
517# Directory traversal and file finding
518#
519
520#
521# Return a sorted list of files found under the specified directory
522#
523def findFiles(d) :
524	for path, subdirs, files in os.walk(d) :
525		files.sort()
526		for name in files :
527			yield os.path.join(path, name)
528
529#
530# Examine all files in base, ptch
531#
532# Return a list of files appearing in both proto areas,
533# a list of new files (files found only in ptch) and
534# a list of deleted files (files found only in base)
535#
536def protoCatalog(base, ptch) :
537
538	compFiles = []		# List of files in both proto areas
539	ptchList = []		# List of file in patch proto area
540
541	newFiles = []		# New files detected
542	deletedFiles = []	# Deleted files
543
544	debug("Getting the list of files in the base area");
545	baseFilesList = list(findFiles(base))
546	baseStringLength = len(base)
547	debug("Found " + str(len(baseFilesList)) + " files")
548
549	debug("Getting the list of files in the patch area");
550	ptchFilesList = list(findFiles(ptch))
551	ptchStringLength = len(ptch)
552	debug("Found " + str(len(ptchFilesList)) + " files")
553
554	# Inventory files in the base proto area
555	debug("Determining the list of regular files in the base area");
556	for fn in baseFilesList :
557		if os.path.islink(fn) :
558			continue
559
560		fileName = fn[baseStringLength:]
561		compFiles.append(fileName)
562	debug("Found " + str(len(compFiles)) + " files")
563
564	# Inventory files in the patch proto area
565	debug("Determining the list of regular files in the patch area");
566	for fn in ptchFilesList :
567		if os.path.islink(fn) :
568			continue
569
570		fileName = fn[ptchStringLength:]
571		ptchList.append(fileName)
572	debug("Found " + str(len(ptchList)) + " files")
573
574	# Deleted files appear in the base area, but not the patch area
575	debug("Searching for deleted files by comparing the lists")
576	for fileName in compFiles :
577		if not fileName in ptchList :
578			deletedFiles.append(fileName)
579	debug("Found " + str(len(deletedFiles)) + " deleted files")
580
581	# Eliminate "deleted" files from the list of objects appearing
582	# in both the base and patch proto areas
583	debug("Eliminating deleted files from the list of objects")
584	for fileName in deletedFiles :
585		try:
586			compFiles.remove(fileName)
587		except:
588			error("filelist.remove() failed")
589	debug("List for comparison reduced to " + str(len(compFiles))
590	    + " files")
591
592	# New files appear in the patch area, but not the base
593	debug("Getting the list of newly added files")
594	for fileName in ptchList :
595		if not fileName in compFiles :
596			newFiles.append(fileName)
597	debug("Found " + str(len(newFiles)) + " new files")
598
599	return compFiles, newFiles, deletedFiles
600
601#
602# Examine the files listed in the input file list
603#
604# Return a list of files appearing in both proto areas,
605# a list of new files (files found only in ptch) and
606# a list of deleted files (files found only in base)
607#
608def flistCatalog(base, ptch, flist) :
609	compFiles = []		# List of files in both proto areas
610	newFiles = []		# New files detected
611	deletedFiles = []	# Deleted files
612
613	try:
614		fd = open(flist, "r")
615	except:
616		error("could not open: " + flist)
617		cleanup(1)
618
619	files = []
620	files = fd.readlines()
621	fd.close()
622
623	for f in files :
624		ptch_present = True
625		base_present = True
626
627		if f == '\n' :
628			continue
629
630		# the fileNames have a trailing '\n'
631		f = f.rstrip()
632
633		# The objects in the file list have paths relative
634		# to $ROOT or to the base/ptch directory specified on
635		# the command line.
636		# If it's relative to $ROOT, we'll need to add back the
637		# root_`uname -p` goo we stripped off in fnFormat()
638		if os.path.exists(base + f) :
639			fn = f;
640		elif os.path.exists(base + "root_" + arch + "/" + f) :
641			fn = "root_" + arch + "/" + f
642		else :
643			base_present = False
644
645		if base_present :
646			if not os.path.exists(ptch + fn) :
647				ptch_present = False
648		else :
649			if os.path.exists(ptch + f) :
650				fn = f
651			elif os.path.exists(ptch + "root_" + arch + "/" + f) :
652				fn = "root_" + arch + "/" + f
653			else :
654				ptch_present = False
655
656		if os.path.islink(base + fn) :	# ignore links
657			base_present = False
658		if os.path.islink(ptch + fn) :
659			ptch_present = False
660
661		if base_present and ptch_present :
662			compFiles.append(fn)
663		elif base_present :
664			deletedFiles.append(fn)
665		elif ptch_present :
666			newFiles.append(fn)
667		else :
668			if (os.path.islink(base + fn) and
669			    os.path.islink(ptch + fn)) :
670				continue
671			error(f + " in file list, but not in either tree. " +
672			    "Skipping...")
673
674	return compFiles, newFiles, deletedFiles
675
676
677#
678# Build a fully qualified path to an external tool/utility.
679# Consider the default system locations. For onbld tools, if
680# the -t option was specified, we'll try to use built tools in $SRC tools,
681# and otherwise, we'll fall back on /opt/onbld/
682#
683def find_tool(tool) :
684
685	# First, check what was passed
686	if os.path.exists(tool) :
687		return tool
688
689	# Next try in wsdiff path
690	for pdir in wsdiff_path :
691		location = pdir + "/" + tool
692		if os.path.exists(location) :
693			return location + " "
694
695		location = pdir + "/" + arch + "/" + tool
696		if os.path.exists(location) :
697			return location + " "
698
699	error("Could not find path to: " + tool);
700	sys.exit(1);
701
702
703#####
704# ELF file comparison helper routines
705#
706
707#
708# Return a dictionary of ELF section types keyed by section name
709#
710def get_elfheader(f) :
711
712	header = {}
713
714	rc, hstring = getoutput(elfdump_cmd + " -c " + f)
715
716	if len(hstring) == 0 :
717		error("Failed to dump ELF header for " + f)
718		raise
719		return
720
721	# elfdump(1) dumps the section headers with the section name
722	# following "sh_name:", and the section type following "sh_type:"
723	sections = hstring.split("Section Header")
724	for sect in sections :
725		datap = sect.find("sh_name:");
726		if datap == -1 :
727			continue
728		section = sect[datap:].split()[1]
729		datap = sect.find("sh_type:");
730		if datap == -1 :
731			error("Could not get type for sect: " + section +
732			      " in " + f)
733		sh_type = sect[datap:].split()[2]
734		header[section] = sh_type
735
736	return header
737
738#
739# Extract data in the specified ELF section from the given file
740#
741def extract_elf_section(f, section) :
742
743	rc, data = getoutput(dump_cmd + " -sn " + section + " " + f)
744
745	if len(data) == 0 :
746		error(dump_cmd + "yielded no data on section " + section +
747		    " of " + f)
748		raise
749		return
750
751	# dump(1) displays the file name to start...
752	# get past it to the data itself
753	dbegin = data.find(":") + 1
754	data = data[dbegin:];
755
756	return (data)
757
758#
759# Return a (hopefully meaningful) human readable set of diffs
760# for the specified ELF section between f1 and f2
761#
762# Depending on the section, various means for dumping and diffing
763# the data may be employed.
764#
765
766text_sections = [ '.text', '.init', '.fini' ]
767
768# Helper to generate the requireed commands for diffing two .SUNW_ctf
769# sections.
770def diff_ctf(f1, f2):
771
772	# Find genunix so that it can be used for parent CTF data when
773	# appropriate.
774	if diff_ctf.genunix1 is None:
775		global genunix, baseWsRoot, ptchWsRoot
776
777		if (baseWsRoot and ptchWsRoot and
778		    os.path.isfile(os.path.join(baseWsRoot, genunix)) and
779		    os.path.isfile(os.path.join(ptchWsRoot, genunix))):
780			diff_ctf.genunix1 = os.path.join(baseWsRoot, genunix)
781			diff_ctf.genunix2 = os.path.join(ptchWsRoot, genunix)
782			debug("CTF: Found {}".format(diff_ctf.genunix1))
783			debug("CTF: Found {}".format(diff_ctf.genunix2))
784		else:
785			# Could not find genunix, do the best we can.
786			error("diff_ctf: Could not find genunix. " +
787			    "CTF diffs will be less useful.")
788			diff_ctf.genunix1 = diff_ctf.genunix2 = False
789
790	# Determine if this is a merged file from genunix by looking
791	# at the parent
792	rc, data = getoutput("{} -h {}".format(ctfdump_cmd, f1))
793	if rc != 0:
794		error("Could not read CTF header: {}".format(data))
795		return (None, None)
796
797	parent = None
798	for line in data.split('\n'):
799		if line.strip().startswith('cth_parname'):
800			try:
801				parent = line.split('=')[1].strip()
802				break
803			except:
804				pass
805
806	cmd1 = cmd2 = "{} -c ".format(ctfdump_cmd)
807	if parent == "genunix":
808		if diff_ctf.genunix1 and diff_ctf.genunix2:
809			cmd1 += "-p {} ".format(diff_ctf.genunix1)
810			cmd2 += "-p {} ".format(diff_ctf.genunix2)
811	elif parent is None or (len(parent) > 0 and parent != "(anon)"):
812		error("Unknown CTF Parent: {}".format(parent))
813		return (None, None)
814
815	cmd1 += f1
816	cmd2 += f2
817
818	return (cmd1, cmd2)
819
820diff_ctf.genunix1 = None
821diff_ctf.genunix2 = None
822
823def diff_elf_section(f1, f2, section, sh_type) :
824
825	t = threading.current_thread()
826	tmpFile1 = tmpDir1 + os.path.basename(f1) + t.name
827	tmpFile2 = tmpDir2 + os.path.basename(f2) + t.name
828
829	if ((sh_type == "SHT_RELA") or
830	    (sh_type == "SHT_REL")): # relocation section
831		cmd1 = elfdump_cmd + " -r " + f1 + " > " + tmpFile1
832		cmd2 = elfdump_cmd + " -r " + f2 + " > " + tmpFile2
833	elif (section == ".group") :
834		cmd1 = elfdump_cmd + " -g " + f1 + " > " + tmpFile1
835		cmd2 = elfdump_cmd + " -g " + f2 + " > " + tmpFile2
836	elif (section == ".hash") :
837		cmd1 = elfdump_cmd + " -h " + f1 + " > " + tmpFile1
838		cmd2 = elfdump_cmd + " -h " + f2 + " > " + tmpFile2
839	elif (section == ".dynamic") :
840		cmd1 = elfdump_cmd + " -d " + f1 + " > " + tmpFile1
841		cmd2 = elfdump_cmd + " -d " + f2 + " > " + tmpFile2
842	elif (section == ".got") :
843		cmd1 = elfdump_cmd + " -G " + f1 + " > " + tmpFile1
844		cmd2 = elfdump_cmd + " -G " + f2 + " > " + tmpFile2
845	elif (section == ".SUNW_cap") :
846		cmd1 = elfdump_cmd + " -H " + f1 + " > " + tmpFile1
847		cmd2 = elfdump_cmd + " -H " + f2 + " > " + tmpFile2
848	elif (section == ".interp") :
849		cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1
850		cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2
851	elif (section == ".symtab" or section == ".dynsym") :
852		cmd1 = (elfdump_cmd + " -s -N " + section + " " + f1 +
853		    " > " + tmpFile1)
854		cmd2 = (elfdump_cmd + " -s -N " + section + " " + f2 +
855		    " > " + tmpFile2)
856	elif (section in text_sections) :
857		# dis sometimes complains when it hits something it doesn't
858		# know how to disassemble. Just ignore it, as the output
859		# being generated here is human readable, and we've already
860		# correctly flagged the difference.
861		cmd1 = (dis_cmd + " -t " + section + " " + f1 +
862		       " 2>/dev/null | grep -v disassembly > " + tmpFile1)
863		cmd2 = (dis_cmd + " -t " + section + " " + f2 +
864		       " 2>/dev/null | grep -v disassembly > " + tmpFile2)
865	elif (section == ".SUNW_ctf"):
866		(cmd1, cmd2) = diff_ctf(f1, f2)
867		if not cmd1:
868			return ""
869		cmd1 += " > {}".format(tmpFile1)
870		cmd2 += " > {}".format(tmpFile2)
871
872	else :
873		cmd1 = (elfdump_cmd + " -w " + tmpFile1 + " -N " +
874		       section + " " + f1)
875		cmd2 = (elfdump_cmd + " -w " + tmpFile2 + " -N " +
876		       section + " " + f2)
877
878	os.system(cmd1)
879	os.system(cmd2)
880
881	data = diffFileData(tmpFile1, tmpFile2)
882
883	# remove temp files as we no longer need them
884	try:
885		os.unlink(tmpFile1)
886		os.unlink(tmpFile2)
887	except OSError as e:
888		error("diff_elf_section: unlink failed {}".format(e))
889
890	return (data)
891
892#
893# compare the relevant sections of two ELF binaries
894# and report any differences
895#
896# Returns: 1 if any differenes found
897#          0 if no differences found
898#	  -1 on error
899#
900
901# Sections deliberately not considered when comparing two ELF
902# binaries. Differences observed in these sections are not considered
903# significant where patch deliverable identification is concerned.
904sections_to_skip = [ ".SUNW_signature",
905		     ".comment",
906		     ".debug",
907		     ".plt",
908		     ".rela.bss",
909		     ".rela.plt",
910		     ".line",
911		     ".note",
912		     ".compcom",
913		     ".SUNW_dof",
914		     ]
915
916sections_preferred = [ ".SUNW_ctf",
917		       ".rodata.str1.8",
918		       ".rodata.str1.1",
919		       ".rodata",
920		       ".data1",
921		       ".data",
922		       ".text",
923		       ]
924
925# Some sections must always be extracted and diffed to check that there are
926# real differences.
927sections_to_always_diff = [ ".SUNW_ctf" ]
928
929def compareElfs(base, ptch, quiet) :
930
931	global logging
932
933	try:
934		base_header = get_elfheader(base)
935	except:
936		return
937	sections = list(base_header.keys())
938	sections.sort()
939
940	try:
941		ptch_header = get_elfheader(ptch)
942	except:
943		return
944	e2_only_sections = list(ptch_header.keys())
945	e2_only_sections.sort()
946
947	e1_only_sections = []
948
949	fileName = fnFormat(base)
950
951	# Derive the list of ELF sections found only in
952	# either e1 or e2.
953	for sect in sections :
954		if not sect in e2_only_sections :
955			e1_only_sections.append(sect)
956		else :
957			e2_only_sections.remove(sect)
958
959	if len(e1_only_sections) > 0 :
960		if quiet :
961			return 1
962
963		data = ""
964		if logging :
965			slist = ""
966			for sect in e1_only_sections :
967				slist = slist + sect + "\t"
968			data = ("ELF sections found in " +
969				base + " but not in " + ptch +
970				"\n\n" + slist)
971
972		difference(fileName, "ELF", data)
973		return 1
974
975	if len(e2_only_sections) > 0 :
976		if quiet :
977			return 1
978
979		data = ""
980		if logging :
981			slist = ""
982			for sect in e2_only_sections :
983				slist = slist + sect + "\t"
984			data = ("ELF sections found in " +
985				ptch + " but not in " + base +
986				"\n\n" + slist)
987
988		difference(fileName, "ELF", data)
989		return 1
990
991	# Look for preferred sections, and put those at the
992	# top of the list of sections to compare
993	for psect in sections_preferred :
994		if psect in sections :
995			sections.remove(psect)
996			sections.insert(0, psect)
997
998	# Compare ELF sections
999	first_section = True
1000	for sect in sections :
1001
1002		if sect in sections_to_skip :
1003			continue
1004
1005		try:
1006			s1 = extract_elf_section(base, sect);
1007		except:
1008			return
1009
1010		try:
1011			s2 = extract_elf_section(ptch, sect);
1012		except:
1013			return
1014
1015		if len(s1) != len (s2) or s1 != s2:
1016			if not quiet or sect in sections_to_always_diff:
1017				sh_type = base_header[sect]
1018				data = diff_elf_section(base, ptch,
1019							sect, sh_type)
1020
1021				if len(data) == 0:
1022					continue # No differences
1023
1024			if not quiet:
1025				# If all ELF sections are being reported, then
1026				# invoke difference() to flag the file name to
1027				# stdout only once. Any other section
1028				# differences should be logged to the results
1029				# file directly
1030				if not first_section :
1031					log_difference(fileName,
1032					    "ELF " + sect, data)
1033				else :
1034					difference(fileName, "ELF " + sect,
1035					    data)
1036
1037			if not reportAllSects :
1038				return 1
1039			first_section = False
1040
1041	return 0
1042
1043#####
1044# recursively remove 2 directories
1045#
1046# Used for removal of temporary directory strucures (ignores any errors).
1047#
1048def clearTmpDirs(dir1, dir2) :
1049
1050	if os.path.isdir(dir1) > 0 :
1051		shutil.rmtree(dir1, True)
1052
1053	if os.path.isdir(dir2) > 0 :
1054		shutil.rmtree(dir2, True)
1055
1056
1057#####
1058# Archive object comparison
1059#
1060# Returns 1 if difference detected
1061#         0 if no difference detected
1062#        -1 on error
1063#
1064def compareArchives(base, ptch, fileType) :
1065
1066	fileName = fnFormat(base)
1067	t = threading.current_thread()
1068	ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.name
1069	ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.name
1070
1071	#
1072	# Be optimistic and first try a straight file compare
1073	# as it will allow us to finish up quickly.
1074	#
1075	if compareBasic(base, ptch, True, fileType) == 0 :
1076		return 0
1077
1078	try:
1079		os.makedirs(ArchTmpDir1)
1080		os.makedirs(ArchTmpDir2)
1081	except OSError as e:
1082		error("compareArchives: makedir failed {}".format(e))
1083		return -1
1084
1085	# copy over the objects to the temp areas, and
1086	# unpack them
1087	baseCmd = "cp -fp " + base + " " + ArchTmpDir1
1088	rc, output = getoutput(baseCmd)
1089	if rc != 0:
1090		error(baseCmd + " failed: " + output)
1091		clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1092		return -1
1093
1094	ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2
1095	rc, output = getoutput(ptchCmd)
1096	if rc != 0:
1097		error(ptchCmd + " failed: " + output)
1098		clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1099		return -1
1100
1101	bname = fileName.split('/')[-1]
1102	if fileType == "Java Archive" :
1103		baseCmd = ("cd " + ArchTmpDir1 + "; " + "jar xf " + bname +
1104			  "; rm -f " + bname + " META-INF/MANIFEST.MF")
1105		ptchCmd = ("cd " + ArchTmpDir2 + "; " + "jar xf " + bname +
1106			  "; rm -f " + bname + " META-INF/MANIFEST.MF")
1107	elif fileType == "ELF Object Archive" :
1108		baseCmd = ("cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " +
1109			  bname + "; rm -f " + bname)
1110		ptchCmd = ("cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " +
1111			  bname + "; rm -f " + bname)
1112	else :
1113		error("unexpected file type: " + fileType)
1114		clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1115		return -1
1116
1117	os.system(baseCmd)
1118	os.system(ptchCmd)
1119
1120	baseFlist = list(findFiles(ArchTmpDir1))
1121	ptchFlist = list(findFiles(ArchTmpDir2))
1122
1123	# Trim leading path off base/ptch file lists
1124	flist = []
1125	for fn in baseFlist :
1126		flist.append(str_prefix_trunc(fn, ArchTmpDir1))
1127	baseFlist = flist
1128
1129	flist = []
1130	for fn in ptchFlist :
1131		flist.append(str_prefix_trunc(fn, ArchTmpDir2))
1132	ptchFlist = flist
1133
1134	for fn in ptchFlist :
1135		if not fn in baseFlist :
1136			difference(fileName, fileType,
1137				   fn + " added to " + fileName)
1138			clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1139			return 1
1140
1141	for fn in baseFlist :
1142		if not fn in ptchFlist :
1143			difference(fileName, fileType,
1144				   fn + " removed from " + fileName)
1145			clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1146			return 1
1147
1148		differs = compareOneFile((ArchTmpDir1 + fn),
1149		    (ArchTmpDir2 + fn), True)
1150		if differs :
1151			difference(fileName, fileType,
1152				   fn + " in " + fileName + " differs")
1153			clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1154			return 1
1155
1156	clearTmpDirs(ArchTmpDir1, ArchTmpDir2)
1157	return 0
1158
1159#####
1160# (Basic) file comparison
1161#
1162# Returns 1 if difference detected
1163#         0 if no difference detected
1164#        -1 on error
1165#
1166def compareBasic(base, ptch, quiet, fileType) :
1167
1168	fileName = fnFormat(base);
1169
1170	if quiet and os.stat(base)[ST_SIZE] != os.stat(ptch)[ST_SIZE] :
1171		return 1
1172
1173	try:
1174		with open(base, 'rb') as fh:
1175			baseData = fh.read()
1176	except:
1177		error("could not open " + base)
1178		return -1
1179
1180	try:
1181		with open(ptch, 'rb') as fh:
1182			ptchData = fh.read()
1183	except:
1184		error("could not open " + ptch)
1185		return -1
1186
1187	if quiet :
1188		if baseData != ptchData :
1189			return 1
1190	else :
1191		if len(baseData) != len(ptchData) or baseData != ptchData :
1192			diffs = diffFileData(base, ptch)
1193			difference(fileName, fileType, diffs)
1194			return 1
1195	return 0
1196
1197
1198#####
1199# Compare two objects by producing a data dump from
1200# each object, and then comparing the dump data
1201#
1202# Returns: 1 if a difference is detected
1203#          0 if no difference detected
1204#         -1 upon error
1205#
1206def compareByDumping(base, ptch, quiet, fileType) :
1207
1208	fileName = fnFormat(base);
1209	t = threading.current_thread()
1210	tmpFile1 = tmpDir1 + os.path.basename(base) + t.name
1211	tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.name
1212
1213	if fileType == "Lint Library" :
1214		baseCmd = (lintdump_cmd + " -ir " + base +
1215			  " | egrep -v '(LINTOBJ|LINTMOD):'" +
1216			  " | grep -v PASS[1-3]:" +
1217			  " > " + tmpFile1)
1218		ptchCmd = (lintdump_cmd + " -ir " + ptch +
1219			  " | egrep -v '(LINTOBJ|LINTMOD):'" +
1220			  " | grep -v PASS[1-3]:" +
1221			  " > " + tmpFile2)
1222	elif fileType == "Sqlite Database" :
1223		baseCmd = ("echo .dump | " + sqlite_cmd + base + " > " +
1224			  tmpFile1)
1225		ptchCmd = ("echo .dump | " + sqlite_cmd + ptch + " > " +
1226			  tmpFile2)
1227
1228	os.system(baseCmd)
1229	os.system(ptchCmd)
1230
1231	try:
1232		with open(tmpFile1, 'rb') as fh:
1233			baseData = fh.read()
1234	except:
1235		error("could not open: " + tmpFile1)
1236		return
1237
1238	try:
1239		with open(tmpFile2, 'rb') as fh:
1240			ptchData = fh.read()
1241	except:
1242		error("could not open: " + tmpFile2)
1243		return
1244
1245	ret = 0
1246
1247	if len(baseData) != len(ptchData) or baseData != ptchData :
1248		if not quiet :
1249			data = diffFileData(tmpFile1, tmpFile2);
1250		ret = 1
1251
1252	# Remove the temporary files now.
1253	try:
1254		os.unlink(tmpFile1)
1255		os.unlink(tmpFile2)
1256	except OSError as e:
1257		error("compareByDumping: unlink failed {}".format(e))
1258
1259	return ret
1260
1261#####
1262#
1263# SIGINT signal handler. Changes thread control variable to tell the threads
1264# to finish their current job and exit.
1265#
1266def discontinue_processing(signl, frme):
1267	global keep_processing
1268
1269	print("Caught Ctrl-C, stopping the threads", file=sys.stderr)
1270	keep_processing = False
1271
1272	return 0
1273
1274#####
1275#
1276# worker thread for changedFiles processing
1277#
1278class workerThread(threading.Thread) :
1279	def run(self):
1280		global wset_lock
1281		global changedFiles
1282		global baseRoot
1283		global ptchRoot
1284		global keep_processing
1285
1286		while (keep_processing) :
1287			# grab the lock to changedFiles and remove one member
1288			# and process it
1289			wset_lock.acquire()
1290			try :
1291				fn = changedFiles.pop()
1292			except IndexError :
1293				# there is nothing more to do
1294				wset_lock.release()
1295				return
1296			wset_lock.release()
1297
1298			base = baseRoot + fn
1299			ptch = ptchRoot + fn
1300
1301			compareOneFile(base, ptch, False)
1302
1303
1304#####
1305# Compare two objects. Detect type changes.
1306# Vector off to the appropriate type specific
1307# compare routine based on the type.
1308#
1309def compareOneFile(base, ptch, quiet) :
1310
1311	# Verify the file types.
1312	# If they are different, indicate this and move on
1313	btype = getTheFileType(base)
1314	ptype = getTheFileType(ptch)
1315
1316	if btype == 'Error' or ptype == 'Error' :
1317		return -1
1318
1319	fileName = fnFormat(base)
1320
1321	if (btype != ptype) :
1322		if not quiet :
1323			difference(fileName, "file type",
1324			    btype + " to " + ptype)
1325		return 1
1326	else :
1327		fileType = btype
1328
1329	if (fileType == 'ELF') :
1330		return compareElfs(base, ptch, quiet)
1331
1332	elif (fileType == 'Java Archive' or fileType == 'ELF Object Archive') :
1333		return compareArchives(base, ptch, fileType)
1334
1335	elif (fileType == 'HTML') :
1336		return compareBasic(base, ptch, quiet, fileType)
1337
1338	elif ( fileType == 'Lint Library' ) :
1339		return compareByDumping(base, ptch, quiet, fileType)
1340
1341	elif ( fileType == 'Sqlite Database' ) :
1342		return compareByDumping(base, ptch, quiet, fileType)
1343
1344	else :
1345		# it has to be some variety of text file
1346		return compareBasic(base, ptch, quiet, fileType)
1347
1348# Cleanup and self-terminate
1349def cleanup(ret) :
1350
1351	debug("Performing cleanup (" + str(ret) + ")")
1352	if os.path.isdir(tmpDir1) > 0 :
1353		shutil.rmtree(tmpDir1)
1354
1355	if os.path.isdir(tmpDir2) > 0 :
1356		shutil.rmtree(tmpDir2)
1357
1358	if logging :
1359		log.close()
1360
1361	sys.exit(ret)
1362
1363def main() :
1364
1365	# Log file handle
1366	global log
1367
1368	# Globals relating to command line options
1369	global logging, vdiffs, reportAllSects
1370
1371	# Named temporary files / directories
1372	global tmpDir1, tmpDir2
1373
1374	# Command paths
1375	global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, \
1376	    diff_cmd, sqlite_cmd, ctfdump_cmd
1377
1378	# Default search path
1379	global wsdiff_path
1380
1381	# Essentially "uname -p"
1382	global arch
1383
1384	# changed files for worker thread processing
1385	global changedFiles
1386
1387	global baseRoot, ptchRoot, baseWsRoot, ptchWsRoot
1388
1389	# Sort the list of files from a temporary file
1390	global o_sorted
1391	global differentFiles
1392
1393	# Debugging indicator
1394	global debugon
1395
1396	# Some globals need to be initialized
1397	debugon = logging = vdiffs = reportAllSects = o_sorted = False
1398
1399	# Process command line arguments
1400	# Return values are returned from args() in alpha order
1401	# (Yes, python functions can return multiple values (ewww))
1402	# Note that args() also set the globals:
1403	#	logging to True if verbose logging (to a file) was enabled
1404	#	vdiffs to True if logged differences aren't to be truncated
1405	#	reportAllSects to True if all ELF section differences are to
1406	#	be reported
1407	#
1408	baseRoot, fileNamesFile, localTools, ptchRoot, results = args()
1409
1410	#
1411	# Set up the results/log file
1412	#
1413	if logging :
1414		try:
1415			log = open(results, "w")
1416		except:
1417			logging = False
1418			error("failed to open log file: {}".format(log))
1419			sys.exit(1)
1420
1421		v_info("# This file was produced by wsdiff")
1422		dateTimeStr= time.strftime('# %Y-%m-%d at %H:%M:%S',
1423		    time.localtime())
1424		v_info(dateTimeStr)
1425
1426	# Changed files (used only for the sorted case)
1427	if o_sorted :
1428		differentFiles = []
1429
1430	#
1431	# Build paths to the tools required tools
1432	#
1433	# Try to look for tools in $SRC/tools if the "-t" option
1434	# was specified
1435	#
1436	rc, arch = getoutput("uname -p")
1437	arch = arch.rstrip()
1438	if localTools :
1439		try:
1440			src = os.environ['SRC']
1441		except:
1442			error("-t specified, but $SRC not set. " +
1443			    "Cannot find $SRC/tools")
1444			src = ""
1445		if len(src) > 0 :
1446			wsdiff_path.insert(0,
1447			    src + "/tools/proto/opt/onbld/bin")
1448
1449	lintdump_cmd = find_tool("lintdump")
1450	elfdump_cmd = find_tool("elfdump")
1451	dump_cmd = find_tool("dump")
1452	od_cmd = find_tool("od")
1453	dis_cmd = find_tool("dis")
1454	diff_cmd = find_tool("diff")
1455	sqlite_cmd = find_tool("sqlite")
1456	ctfdump_cmd = find_tool("ctfdump")
1457
1458	#
1459	# Set resource limit for number of open files as high as possible.
1460	# This might get handy with big number of threads.
1461	#
1462	(nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
1463	try:
1464		resource.setrlimit(resource.RLIMIT_NOFILE,
1465		    (nofile_hard, nofile_hard))
1466	except:
1467		error("cannot set resource limits for number of open files")
1468		sys.exit(1)
1469
1470	#
1471	# validate the base and patch paths
1472	#
1473	baseRoot = os.path.abspath(baseRoot) + os.sep
1474	ptchRoot = os.path.abspath(ptchRoot) + os.sep
1475
1476	if not os.path.exists(baseRoot) :
1477		error("old proto area: {} does not exist".format(baseRoot))
1478		sys.exit(1)
1479
1480	if not os.path.exists(ptchRoot) :
1481		error("new proto area: {} does not exist".format(ptchRoot))
1482		sys.exit(1)
1483
1484	#
1485	# attempt to find the workspace root directory for the proto area
1486	#
1487	baseWsRoot = protoroot(baseRoot)
1488	ptchWsRoot = protoroot(ptchRoot)
1489	debug("base workspace root: {}".format(baseWsRoot))
1490	debug("ptch workspace root: {}".format(ptchWsRoot))
1491
1492	#
1493	# log some information identifying the run
1494	#
1495	v_info("Old proto area: {}".format(baseRoot))
1496	v_info("New proto area: {}".format(ptchRoot))
1497	v_info("Results file: {}".format(results))
1498	v_info("")
1499
1500	#
1501	# Set up the temporary directories / files
1502	# Could use python's tmpdir routines, but these should
1503	# be easier to identify / keep around for debugging
1504	pid = os.getpid()
1505	tmpDir1 = "/tmp/wsdiff_tmp1_" + str(pid) + "/"
1506	tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/"
1507	try:
1508		os.makedirs(tmpDir1)
1509		os.makedirs(tmpDir2)
1510	except OSError as e:
1511		error("main: makedir failed {}".format(e))
1512
1513	# Derive a catalog of new, deleted, and to-be-compared objects
1514	# either from the specified base and patch proto areas, or from
1515	# from an input file list
1516	newOrDeleted = False
1517
1518	if fileNamesFile != "" :
1519		changedFiles, newFiles, deletedFiles = \
1520			      flistCatalog(baseRoot, ptchRoot, fileNamesFile)
1521	else :
1522		changedFiles, newFiles, deletedFiles = \
1523				protoCatalog(baseRoot, ptchRoot)
1524
1525	if len(newFiles) > 0 :
1526		newOrDeleted = True
1527		info("\nNew objects found: ")
1528
1529		if o_sorted :
1530			newFiles.sort()
1531		for fn in newFiles :
1532			info(fnFormat(fn))
1533
1534	if len(deletedFiles) > 0 :
1535		newOrDeleted = True
1536		info("\nObjects removed: ")
1537
1538		if o_sorted :
1539			deletedFiles.sort()
1540		for fn in deletedFiles :
1541			info(fnFormat(fn))
1542
1543	if newOrDeleted :
1544		info("\nChanged objects: ")
1545	if o_sorted :
1546		debug("The list will appear after the processing is done")
1547
1548	# Here's where all the heavy lifting happens
1549	# Perform a comparison on each object appearing in
1550	# both proto areas. compareOneFile will examine the
1551	# file types of each object, and will vector off to
1552	# the appropriate comparison routine, where the compare
1553	# will happen, and any differences will be reported / logged
1554
1555	# determine maximum number of worker threads by using
1556	# DMAKE_MAX_JOBS environment variable set by nightly(1)
1557	# or get number of CPUs in the system
1558	try:
1559		max_threads = int(os.environ['DMAKE_MAX_JOBS'])
1560	except:
1561		max_threads = os.sysconf("SC_NPROCESSORS_ONLN")
1562		# If we cannot get number of online CPUs in the system
1563		# run unparallelized otherwise bump the number up 20%
1564		# to achieve best results.
1565		if max_threads == -1 :
1566			max_threads = 1
1567		else :
1568			max_threads += int(max_threads/5)
1569
1570	# Set signal handler to attempt graceful exit
1571	debug("Setting signal handler")
1572	signal.signal( signal.SIGINT, discontinue_processing )
1573
1574	# Create and unleash the threads
1575	# Only at most max_threads must be running at any moment
1576	mythreads = []
1577	debug("Spawning " + str(max_threads) + " threads");
1578	for i in range(max_threads) :
1579		thread = workerThread()
1580		mythreads.append(thread)
1581		mythreads[i].start()
1582
1583	# Wait for the threads to finish and do cleanup if interrupted
1584	debug("Waiting for the threads to finish")
1585	while True:
1586		if not True in [thread.is_alive() for thread in mythreads]:
1587		    break
1588		else:
1589		    # Some threads are still going
1590		    time.sleep(1)
1591
1592	# Interrupted by SIGINT
1593	if keep_processing == False :
1594		cleanup(1)
1595
1596	# If the list of differences was sorted it is stored in an array
1597	if o_sorted :
1598		differentFiles.sort()
1599		for f in differentFiles :
1600			info(fnFormat(f))
1601
1602	# We're done, cleanup.
1603	cleanup(0)
1604
1605if __name__ == '__main__' :
1606	try:
1607		main()
1608	except KeyboardInterrupt :
1609		cleanup(1);
1610