1#! /usr/bin/python 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2008 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27 28# 29# Various database lookup classes/methods, i.e.: 30# * monaco 31# * bugs.opensolaris.org (b.o.o.) 32# * opensolaris.org/cgi/arc.py (for ARC) 33# 34 35import re 36import urllib 37import urllib2 38import htmllib 39import os 40from socket import socket, AF_INET, SOCK_STREAM 41 42from onbld.Checks import onSWAN 43 44class BugException(Exception): 45 def __init__(self, data=''): 46 self.data = data 47 Exception.__init__(self, data) 48 49 def __str__(self): 50 return "Unknown error: %s" % self.data 51 52class NonExistentBug(BugException): 53 def __str__(self): 54 return "Bug %s does not exist" % self.data 55 56class BugDBException(Exception): 57 def __init__(self, data=''): 58 self.data = data 59 Exception.__init__(self, data) 60 61 def __str__(self): 62 return "Unknown bug database: %s" % self.data 63 64class BugDB(object): 65 """Lookup change requests. 66 67 Object can be used on or off of SWAN, using either monaco or 68 bugs.opensolaris.org as a database. 69 70 Usage: 71 bdb = BugDB() 72 r = bdb.lookup("6455550") 73 print r["6455550"]["synopsis"] 74 r = bdb.lookup(["6455550", "6505625"]) 75 print r["6505625"]["synopsis"] 76 """ 77 78 def __init__(self, priority = ("bugster",), forceBoo = False): 79 """Create a BugDB object. 80 81 Keyword argument: 82 forceBoo: use b.o.o even from SWAN (default=False) 83 priority: use bug databases in this order 84 """ 85 self.__validBugDB = ["bugster"] 86 self.__onSWAN = not forceBoo and onSWAN() 87 for database in priority: 88 if database not in self.__validBugDB: 89 raise BugDBException, database 90 self.__priority = priority 91 92 93 def __boobug(self, cr): 94 cr = str(cr) 95 url = "http://bugs.opensolaris.org/view_bug.do" 96 req = urllib2.Request(url, urllib.urlencode({"bug_id": cr})) 97 results = {} 98 try: 99 data = urllib2.urlopen(req).readlines() 100 except urllib2.HTTPError, e: 101 if e.code != 404: 102 print "ERROR: HTTP error at " + \ 103 req.get_full_url() + \ 104 " got error: " + str(e.code) 105 raise e 106 else: 107 raise NonExistentBug 108 except urllib2.URLError, e: 109 print "ERROR: could not connect to " + \ 110 req.get_full_url() + \ 111 ' got error: "' + e.reason[1] + '"' 112 raise e 113 htmlParser = htmllib.HTMLParser(None) 114 metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$') 115 for line in data: 116 m = metaHtmlRe.search(line) 117 if not m: 118 continue 119 val = urllib.unquote(m.group(2)) 120 htmlParser.save_bgn() 121 htmlParser.feed(val) 122 results[m.group(1)] = htmlParser.save_end() 123 htmlParser.close() 124 125 if "synopsis" not in results: 126 raise NonExistentBug(cr) 127 128 results["cr_number"] = cr 129 results["sub_category"] = results.pop("subcategory") 130 results["status"] = results.pop("state") 131 results["date_submitted"] = results.pop("submit_date") 132 133 return results 134 135 136 def __monaco(self, crs): 137 """Return all info for requested change reports. 138 139 Argument: 140 crs: list of change request ids 141 142 Returns: 143 Dictionary, mapping CR=>dictionary, where the nested dictionary 144 is a mapping of field=>value 145 """ 146 147 # 148 # We request synopsis last, and split on only 149 # the number of separators that we expect to 150 # see such that a | in the synopsis doesn't 151 # throw us out of whack. 152 # 153 monacoFields = [ "cr_number", "category", "sub_category", 154 "area", "release", "build", "responsible_manager", 155 "responsible_engineer", "priority", "status", "sub_status", 156 "submitted_by", "date_submitted", "synopsis" ] 157 cmd = [] 158 cmd.append("set What = cr." + ', cr.'.join(monacoFields)) 159 cmd.append("") 160 cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")") 161 cmd.append("") 162 cmd.append("set FinalClauses = order by cr.cr_number") 163 cmd.append("") 164 cmd.append("doMeta genQuery cr") 165 url = "http://hestia.sfbay.sun.com/cgi-bin/expert?format=" 166 url += "Pipe-delimited+text;Go=2;no_header=on;cmds=" 167 url += urllib.quote_plus("\n".join(cmd)) 168 results = {} 169 try: 170 data = urllib2.urlopen(url).readlines() 171 except urllib2.HTTPError, e: 172 print "ERROR: HTTP error at " + url + \ 173 " got error: " + str(e.code) 174 raise e 175 176 except urllib2.URLError, e: 177 print "ERROR: could not connect to " + url + \ 178 ' got error: "' + e.reason[1] + '"' 179 raise e 180 for line in data: 181 line = line.rstrip('\n') 182 values = line.split('|', len(monacoFields) - 1) 183 v = 0 184 cr = values[0] 185 results[cr] = {} 186 for field in monacoFields: 187 results[cr][field] = values[v] 188 v += 1 189 return results 190 191 def lookup(self, crs): 192 """Return all info for requested change reports. 193 194 Argument: 195 crs: one change request id (may be integer, string, or list), 196 or multiple change request ids (must be a list) 197 198 Returns: 199 Dictionary, mapping CR=>dictionary, where the nested dictionary 200 is a mapping of field=>value 201 """ 202 results = {} 203 if not isinstance(crs, list): 204 crs = [str(crs)] 205 for database in self.__priority: 206 if database == "bugster": 207 if self.__onSWAN: 208 results.update(self.__monaco(crs)) 209 # else we're off-swan and querying via boo, which we can 210 # only do one bug at a time 211 else: 212 for cr in crs: 213 cr = str(cr) 214 try: 215 results[cr] = self.__boobug(cr) 216 except NonExistentBug: 217 continue 218 219 # the CR has already been found by one bug database 220 # so don't bother looking it up in the others 221 for cr in crs: 222 if cr in results: 223 crs.remove(cr) 224 225 return results 226#################################################################### 227def ARC(arclist): 228 opts = {} 229 url = "http://opensolaris.org/cgi/arc.py" 230 opts["n"] = str(len(arclist)) 231 for i, arc in enumerate(arclist): 232 arc, case = arc 233 opts["arc" + str(i)] = arc 234 opts["case" + str(i)] = case 235 req = urllib2.Request(url, urllib.urlencode(opts)) 236 try: 237 data = urllib2.urlopen(req).readlines() 238 except urllib2.HTTPError, e: 239 print "ERROR: HTTP error at " + req.get_ful_url() + \ 240 " got error: " + str(e.code) 241 raise e 242 243 except urllib2.URLError, e: 244 print "ERROR: could not connect to " + req.get_ful_url() + \ 245 ' got error: "' + e.reason[1] + '"' 246 raise e 247 ret = {} 248 for line in data: 249 oneline = line.rstrip('\n') 250 fields = oneline.split('|') 251 # check if each is valid ( fields[0]::validity ) 252 if fields[0] != "0": 253 continue 254 arc, case = fields[1].split(" ") 255 ret[(arc, case)] = fields[2] 256 return ret 257 258#################################################################### 259 260# Pointers to the webrti server hostname & port to use 261# Using it directly is probably not *officially* supported, so we'll 262# have a pointer to the official `webrticli` command line interface 263# if using a direct socket connection fails for some reason, so we 264# have a fallback 265WEBRTI_HOST = 'webrti.sfbay.sun.com' 266WEBRTI_PORT = 9188 267WEBRTICLI = '/net/onnv.sfbay.sun.com/export/onnv-gate/public/bin/webrticli' 268 269 270class RtiException(Exception): 271 def __init__(self, data=''): 272 self.data = data 273 Exception.__init__(self, data) 274 275 def __str__(self): 276 return "Unknown error: %s" % self.data 277 278class RtiCallFailed(RtiException): 279 def __str__(self): 280 return "Unable to call webrti: %s" % self.data 281 282class RtiSystemProblem(RtiException): 283 def __str__(self): 284 return "RTI status cannot be determined: %s" % self.data 285 286class RtiIncorrectCR(RtiException): 287 def __str__(self): 288 return "Incorrect CR number specified: %s" % self.data 289 290class RtiNotFound(RtiException): 291 def __str__(self): 292 return "RTI not found: %s" % self.data 293 294class RtiNeedConsolidation(RtiException): 295 def __str__(self): 296 return "More than one consolidation has this CR: %s" % self.data 297 298class RtiBadGate(RtiException): 299 def __str__(self): 300 return "Incorrect gate name specified: %s" % self.data 301 302class RtiOffSwan(RtiException): 303 def __str__(self): 304 return "RTI status checks need SWAN access: %s" % self.data 305 306WEBRTI_ERRORS = { 307 '1': RtiSystemProblem, 308 '2': RtiIncorrectCR, 309 '3': RtiNotFound, 310 '4': RtiNeedConsolidation, 311 '5': RtiBadGate, 312} 313 314# Our Rti object which we'll use to represent an Rti query 315# It's really just a wrapper around the Rti connection, and attempts 316# to establish a direct socket connection and query the webrti server 317# directly (thus avoiding a system/fork/exec call). If it fails, it 318# falls back to the webrticli command line client. 319 320returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)') 321class Rti: 322 """Lookup an RTI. 323 324 Usage: 325 r = Rti("6640538") 326 print r.rtiNumber(); 327 """ 328 329 def __init__(self, cr, gate=None, consolidation=None): 330 """Create an Rti object for the specified change request. 331 332 Argument: 333 cr: change request id 334 335 Keyword arguments, to limit scope of RTI search: 336 gate: path to gate workspace (default=None) 337 consolidation: consolidation name (default=None) 338 """ 339 340 bufSz = 1024 341 addr = (WEBRTI_HOST, WEBRTI_PORT) 342 # If the passed 'cr' was given as an int, then wrap it 343 # into a string to make our life easier 344 if isinstance(cr, int): 345 cr = str(cr) 346 self.__queryCr = cr 347 self.__queryGate = gate 348 self.__queryConsolidation = consolidation 349 350 self.__webRtiOutput = [] 351 self.__mainCR = [] 352 self.__rtiNumber = [] 353 self.__consolidation = [] 354 self.__project = [] 355 self.__status = [] 356 self.__rtiType = [] 357 try: 358 # try to use a direct connection to the 359 # webrti server first 360 sock = socket(AF_INET, SOCK_STREAM) 361 sock.connect(addr) 362 command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr 363 if consolidation: 364 command += "-c\n%s\n" % consolidation 365 if gate: 366 command += "-g\n%s\n" % gate 367 command += "\n" 368 sock.send(command) 369 dataList = [] 370 # keep receiving data from the socket until the 371 # server closes the connection 372 stillReceiving = True 373 while stillReceiving: 374 dataPiece = sock.recv(bufSz) 375 if dataPiece: 376 dataList.append(dataPiece) 377 else: 378 stillReceiving = False 379 # create the lines, skipping the first 380 # ("WEBRTCLI/1.0\n") 381 data = '\n'.join(''.join(dataList).split('\n')[1:]) 382 except: 383 if not onSWAN(): 384 raise RtiOffSwan(cr) 385 386 if not os.path.exists(WEBRTICLI): 387 raise RtiCallFailed('not found') 388 389 # fallback to the "supported" webrticli interface 390 command = WEBRTICLI 391 if consolidation: 392 command += " -c " + consolidation 393 if gate: 394 command += " -g " + gate 395 command += " RTIstatus " + cr 396 397 try: 398 cliPipe = os.popen(command) 399 except: 400 # we couldn't call the webrticli for some 401 # reason, so return a failure 402 raise RtiCallFailed('unknown') 403 404 data = cliPipe.readline() 405 406 # parse the data to see if we got a return code 407 # if we did, then that's bad. if we didn't, 408 # then our call was successfully 409 m = returnCodeRe.search(data) 410 if m: 411 # we got a return code, set it in our 412 # object, set the webRtiOutput for debugging 413 # or logging, and return a failure 414 if m.group(1) in WEBRTI_ERRORS: 415 exc = WEBRTI_ERRORS[m.group(1)] 416 else: 417 exc = RtiException 418 raise exc(data) 419 420 data = data.splitlines() 421 # At this point, we should have valid data 422 for line in data: 423 line = line.rstrip('\r\n') 424 self.__webRtiOutput.append(line) 425 fields = line.split(':') 426 self.__mainCR.append(fields[0]) 427 self.__rtiNumber.append(fields[1]) 428 self.__consolidation.append(fields[2]) 429 self.__project.append(fields[3]) 430 self.__status.append(fields[4]) 431 self.__rtiType.append(fields[5]) 432 433 # accessors in case callers need the raw data 434 def mainCR(self): 435 return self.__mainCR 436 def rtiNumber(self): 437 return self.__rtiNumber 438 def consolidation(self): 439 return self.__consolidation 440 def project(self): 441 return self.__project 442 def status(self): 443 return self.__status 444 def rtiType(self): 445 return self.__rtiType 446 def queryCr(self): 447 return self.__queryCr 448 def queryGate(self): 449 return self.__queryGate 450 def queryConsolidation(self): 451 return self.__queryConsolidation 452 453 # in practice, most callers only care about the following 454 def accepted(self): 455 for status in self.__status: 456 if status != "S_ACCEPTED": 457 return False 458 return True 459 460 # for logging/debugging in case the caller wants the raw webrti output 461 def webRtiOutput(self): 462 return self.__webRtiOutput 463 464