1#! /usr/bin/python 2 3import os 4import sys 5import glob 6import optparse 7import tempfile 8import logging 9import shutil 10import ConfigParser 11 12def data_equal(a, b): 13 # Allow multiple values in assignment separated by '|' 14 a_list = a.split('|') 15 b_list = b.split('|') 16 17 for a_item in a_list: 18 for b_item in b_list: 19 if (a_item == b_item): 20 return True 21 elif (a_item == '*') or (b_item == '*'): 22 return True 23 24 return False 25 26class Fail(Exception): 27 def __init__(self, test, msg): 28 self.msg = msg 29 self.test = test 30 def getMsg(self): 31 return '\'%s\' - %s' % (self.test.path, self.msg) 32 33class Notest(Exception): 34 def __init__(self, test, arch): 35 self.arch = arch 36 self.test = test 37 def getMsg(self): 38 return '[%s] \'%s\'' % (self.arch, self.test.path) 39 40class Unsup(Exception): 41 def __init__(self, test): 42 self.test = test 43 def getMsg(self): 44 return '\'%s\'' % self.test.path 45 46class Event(dict): 47 terms = [ 48 'cpu', 49 'flags', 50 'type', 51 'size', 52 'config', 53 'sample_period', 54 'sample_type', 55 'read_format', 56 'disabled', 57 'inherit', 58 'pinned', 59 'exclusive', 60 'exclude_user', 61 'exclude_kernel', 62 'exclude_hv', 63 'exclude_idle', 64 'mmap', 65 'comm', 66 'freq', 67 'inherit_stat', 68 'enable_on_exec', 69 'task', 70 'watermark', 71 'precise_ip', 72 'mmap_data', 73 'sample_id_all', 74 'exclude_host', 75 'exclude_guest', 76 'exclude_callchain_kernel', 77 'exclude_callchain_user', 78 'wakeup_events', 79 'bp_type', 80 'config1', 81 'config2', 82 'branch_sample_type', 83 'sample_regs_user', 84 'sample_stack_user', 85 ] 86 87 def add(self, data): 88 for key, val in data: 89 log.debug(" %s = %s" % (key, val)) 90 self[key] = val 91 92 def __init__(self, name, data, base): 93 log.debug(" Event %s" % name); 94 self.name = name; 95 self.group = '' 96 self.add(base) 97 self.add(data) 98 99 def equal(self, other): 100 for t in Event.terms: 101 log.debug(" [%s] %s %s" % (t, self[t], other[t])); 102 if not self.has_key(t) or not other.has_key(t): 103 return False 104 if not data_equal(self[t], other[t]): 105 return False 106 return True 107 108 def optional(self): 109 if self.has_key('optional') and self['optional'] == '1': 110 return True 111 return False 112 113 def diff(self, other): 114 for t in Event.terms: 115 if not self.has_key(t) or not other.has_key(t): 116 continue 117 if not data_equal(self[t], other[t]): 118 log.warning("expected %s=%s, got %s" % (t, self[t], other[t])) 119 120# Test file description needs to have following sections: 121# [config] 122# - just single instance in file 123# - needs to specify: 124# 'command' - perf command name 125# 'args' - special command arguments 126# 'ret' - expected command return value (0 by default) 127# 'arch' - architecture specific test (optional) 128# comma separated list, ! at the beginning 129# negates it. 130# 131# [eventX:base] 132# - one or multiple instances in file 133# - expected values assignments 134class Test(object): 135 def __init__(self, path, options): 136 parser = ConfigParser.SafeConfigParser() 137 parser.read(path) 138 139 log.warning("running '%s'" % path) 140 141 self.path = path 142 self.test_dir = options.test_dir 143 self.perf = options.perf 144 self.command = parser.get('config', 'command') 145 self.args = parser.get('config', 'args') 146 147 try: 148 self.ret = parser.get('config', 'ret') 149 except: 150 self.ret = 0 151 152 try: 153 self.arch = parser.get('config', 'arch') 154 log.warning("test limitation '%s'" % self.arch) 155 except: 156 self.arch = '' 157 158 self.expect = {} 159 self.result = {} 160 log.debug(" loading expected events"); 161 self.load_events(path, self.expect) 162 163 def is_event(self, name): 164 if name.find("event") == -1: 165 return False 166 else: 167 return True 168 169 def skip_test(self, myarch): 170 # If architecture not set always run test 171 if self.arch == '': 172 # log.warning("test for arch %s is ok" % myarch) 173 return False 174 175 # Allow multiple values in assignment separated by ',' 176 arch_list = self.arch.split(',') 177 178 # Handle negated list such as !s390x,ppc 179 if arch_list[0][0] == '!': 180 arch_list[0] = arch_list[0][1:] 181 log.warning("excluded architecture list %s" % arch_list) 182 for arch_item in arch_list: 183 # log.warning("test for %s arch is %s" % (arch_item, myarch)) 184 if arch_item == myarch: 185 return True 186 return False 187 188 for arch_item in arch_list: 189 # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch)) 190 if arch_item == myarch: 191 return False 192 return True 193 194 def load_events(self, path, events): 195 parser_event = ConfigParser.SafeConfigParser() 196 parser_event.read(path) 197 198 # The event record section header contains 'event' word, 199 # optionaly followed by ':' allowing to load 'parent 200 # event' first as a base 201 for section in filter(self.is_event, parser_event.sections()): 202 203 parser_items = parser_event.items(section); 204 base_items = {} 205 206 # Read parent event if there's any 207 if (':' in section): 208 base = section[section.index(':') + 1:] 209 parser_base = ConfigParser.SafeConfigParser() 210 parser_base.read(self.test_dir + '/' + base) 211 base_items = parser_base.items('event') 212 213 e = Event(section, parser_items, base_items) 214 events[section] = e 215 216 def run_cmd(self, tempdir): 217 junk1, junk2, junk3, junk4, myarch = (os.uname()) 218 219 if self.skip_test(myarch): 220 raise Notest(self, myarch) 221 222 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir, 223 self.perf, self.command, tempdir, self.args) 224 ret = os.WEXITSTATUS(os.system(cmd)) 225 226 log.info(" '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret))) 227 228 if not data_equal(str(ret), str(self.ret)): 229 raise Unsup(self) 230 231 def compare(self, expect, result): 232 match = {} 233 234 log.debug(" compare"); 235 236 # For each expected event find all matching 237 # events in result. Fail if there's not any. 238 for exp_name, exp_event in expect.items(): 239 exp_list = [] 240 log.debug(" matching [%s]" % exp_name) 241 for res_name, res_event in result.items(): 242 log.debug(" to [%s]" % res_name) 243 if (exp_event.equal(res_event)): 244 exp_list.append(res_name) 245 log.debug(" ->OK") 246 else: 247 log.debug(" ->FAIL"); 248 249 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list))) 250 251 # we did not any matching event - fail 252 if not exp_list: 253 if exp_event.optional(): 254 log.debug(" %s does not match, but is optional" % exp_name) 255 else: 256 exp_event.diff(res_event) 257 raise Fail(self, 'match failure'); 258 259 match[exp_name] = exp_list 260 261 # For each defined group in the expected events 262 # check we match the same group in the result. 263 for exp_name, exp_event in expect.items(): 264 group = exp_event.group 265 266 if (group == ''): 267 continue 268 269 for res_name in match[exp_name]: 270 res_group = result[res_name].group 271 if res_group not in match[group]: 272 raise Fail(self, 'group failure') 273 274 log.debug(" group: [%s] matches group leader %s" % 275 (exp_name, str(match[group]))) 276 277 log.debug(" matched") 278 279 def resolve_groups(self, events): 280 for name, event in events.items(): 281 group_fd = event['group_fd']; 282 if group_fd == '-1': 283 continue; 284 285 for iname, ievent in events.items(): 286 if (ievent['fd'] == group_fd): 287 event.group = iname 288 log.debug('[%s] has group leader [%s]' % (name, iname)) 289 break; 290 291 def run(self): 292 tempdir = tempfile.mkdtemp(); 293 294 try: 295 # run the test script 296 self.run_cmd(tempdir); 297 298 # load events expectation for the test 299 log.debug(" loading result events"); 300 for f in glob.glob(tempdir + '/event*'): 301 self.load_events(f, self.result); 302 303 # resolve group_fd to event names 304 self.resolve_groups(self.expect); 305 self.resolve_groups(self.result); 306 307 # do the expectation - results matching - both ways 308 self.compare(self.expect, self.result) 309 self.compare(self.result, self.expect) 310 311 finally: 312 # cleanup 313 shutil.rmtree(tempdir) 314 315 316def run_tests(options): 317 for f in glob.glob(options.test_dir + '/' + options.test): 318 try: 319 Test(f, options).run() 320 except Unsup, obj: 321 log.warning("unsupp %s" % obj.getMsg()) 322 except Notest, obj: 323 log.warning("skipped %s" % obj.getMsg()) 324 325def setup_log(verbose): 326 global log 327 level = logging.CRITICAL 328 329 if verbose == 1: 330 level = logging.WARNING 331 if verbose == 2: 332 level = logging.INFO 333 if verbose >= 3: 334 level = logging.DEBUG 335 336 log = logging.getLogger('test') 337 log.setLevel(level) 338 ch = logging.StreamHandler() 339 ch.setLevel(level) 340 formatter = logging.Formatter('%(message)s') 341 ch.setFormatter(formatter) 342 log.addHandler(ch) 343 344USAGE = '''%s [OPTIONS] 345 -d dir # tests dir 346 -p path # perf binary 347 -t test # single test 348 -v # verbose level 349''' % sys.argv[0] 350 351def main(): 352 parser = optparse.OptionParser(usage=USAGE) 353 354 parser.add_option("-t", "--test", 355 action="store", type="string", dest="test") 356 parser.add_option("-d", "--test-dir", 357 action="store", type="string", dest="test_dir") 358 parser.add_option("-p", "--perf", 359 action="store", type="string", dest="perf") 360 parser.add_option("-v", "--verbose", 361 action="count", dest="verbose") 362 363 options, args = parser.parse_args() 364 if args: 365 parser.error('FAILED wrong arguments %s' % ' '.join(args)) 366 return -1 367 368 setup_log(options.verbose) 369 370 if not options.test_dir: 371 print 'FAILED no -d option specified' 372 sys.exit(-1) 373 374 if not options.test: 375 options.test = 'test*' 376 377 try: 378 run_tests(options) 379 380 except Fail, obj: 381 print "FAILED %s" % obj.getMsg(); 382 sys.exit(-1) 383 384 sys.exit(0) 385 386if __name__ == '__main__': 387 main() 388