1#! /usr/bin/python 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27 28# 29# Various database lookup classes/methods, i.e.: 30# * monaco 31# * bugs.opensolaris.org (b.o.o.) 32# * arc.opensolaris.org/cgi-bin/arc.cgi (for ARC) 33# 34 35import csv 36import re 37import urllib 38import urllib2 39import htmllib 40import os 41from socket import socket, AF_INET, SOCK_STREAM 42 43from onbld.Checks import onSWAN 44 45class NonExistentBug(Exception): 46 def __str__(self): 47 return "Bug %s does not exist" % (Exception.__str__(self)) 48 49class BugDBException(Exception): 50 def __str__(self): 51 return "Unknown bug database: %s" % (Exception.__str__(self)) 52 53class BugDB(object): 54 """Lookup change requests. 55 56 Object can be used on or off of SWAN, using either monaco or 57 bugs.opensolaris.org as a database. 58 59 Usage: 60 bdb = BugDB() 61 r = bdb.lookup("6455550") 62 print r["6455550"]["synopsis"] 63 r = bdb.lookup(["6455550", "6505625"]) 64 print r["6505625"]["synopsis"] 65 """ 66 67 def __init__(self, priority = ("bugster",), forceBoo=False): 68 """Create a BugDB object. 69 70 Keyword argument: 71 forceBoo: use b.o.o even from SWAN (default=False) 72 priority: use bug databases in this order 73 """ 74 self.__validBugDB = ["bugster"] 75 self.__onSWAN = not forceBoo and onSWAN() 76 for database in priority: 77 if database not in self.__validBugDB: 78 raise BugDBException, database 79 self.__priority = priority 80 81 82 def __boobug(self, cr): 83 cr = str(cr) 84 url = "http://bugs.opensolaris.org/view_bug.do" 85 req = urllib2.Request(url, urllib.urlencode({"bug_id": cr})) 86 results = {} 87 try: 88 data = urllib2.urlopen(req).readlines() 89 except urllib2.HTTPError, e: 90 if e.code != 404: 91 print "ERROR: HTTP error at " + \ 92 req.get_full_url() + \ 93 " got error: " + str(e.code) 94 raise e 95 else: 96 raise NonExistentBug 97 except urllib2.URLError, e: 98 print "ERROR: could not connect to " + \ 99 req.get_full_url() + \ 100 ' got error: "' + e.reason[1] + '"' 101 raise e 102 htmlParser = htmllib.HTMLParser(None) 103 metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$') 104 for line in data: 105 m = metaHtmlRe.search(line) 106 if not m: 107 continue 108 val = urllib.unquote(m.group(2)) 109 htmlParser.save_bgn() 110 htmlParser.feed(val) 111 results[m.group(1)] = htmlParser.save_end() 112 htmlParser.close() 113 114 if "synopsis" not in results: 115 raise NonExistentBug(cr) 116 117 results["cr_number"] = cr 118 results["sub_category"] = results.pop("subcategory") 119 results["status"] = results.pop("state") 120 results["date_submitted"] = results.pop("submit_date") 121 122 return results 123 124 125 def __monaco(self, crs): 126 """Return all info for requested change reports. 127 128 Argument: 129 crs: list of change request ids 130 131 Returns: 132 Dictionary, mapping CR=>dictionary, where the nested dictionary 133 is a mapping of field=>value 134 """ 135 136 # 137 # We request synopsis last, and split on only 138 # the number of separators that we expect to 139 # see such that a | in the synopsis doesn't 140 # throw us out of whack. 141 # 142 monacoFields = [ "cr_number", "category", "sub_category", 143 "area", "release", "build", "responsible_manager", 144 "responsible_engineer", "priority", "status", "sub_status", 145 "submitted_by", "date_submitted", "synopsis" ] 146 cmd = [] 147 cmd.append("set What = cr." + ', cr.'.join(monacoFields)) 148 cmd.append("") 149 cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")") 150 cmd.append("") 151 cmd.append("set FinalClauses = order by cr.cr_number") 152 cmd.append("") 153 cmd.append("doMeta genQuery cr") 154 url = "http://hestia.sfbay.sun.com/cgi-bin/expert?format=" 155 url += "Pipe-delimited+text;Go=2;no_header=on;cmds=" 156 url += urllib.quote_plus("\n".join(cmd)) 157 results = {} 158 try: 159 data = urllib2.urlopen(url).readlines() 160 except urllib2.HTTPError, e: 161 print "ERROR: HTTP error at " + url + \ 162 " got error: " + str(e.code) 163 raise e 164 165 except urllib2.URLError, e: 166 print "ERROR: could not connect to " + url + \ 167 ' got error: "' + e.reason[1] + '"' 168 raise e 169 for line in data: 170 line = line.rstrip('\n') 171 values = line.split('|', len(monacoFields) - 1) 172 v = 0 173 cr = values[0] 174 results[cr] = {} 175 for field in monacoFields: 176 results[cr][field] = values[v] 177 v += 1 178 return results 179 180 def lookup(self, crs): 181 """Return all info for requested change reports. 182 183 Argument: 184 crs: one change request id (may be integer, string, or list), 185 or multiple change request ids (must be a list) 186 187 Returns: 188 Dictionary, mapping CR=>dictionary, where the nested dictionary 189 is a mapping of field=>value 190 """ 191 results = {} 192 if not isinstance(crs, list): 193 crs = [str(crs)] 194 for database in self.__priority: 195 if database == "bugster": 196 if self.__onSWAN: 197 results.update(self.__monaco(crs)) 198 # else we're off-swan and querying via boo, which we can 199 # only do one bug at a time 200 else: 201 for cr in crs: 202 cr = str(cr) 203 try: 204 results[cr] = self.__boobug(cr) 205 except NonExistentBug: 206 continue 207 208 # the CR has already been found by one bug database 209 # so don't bother looking it up in the others 210 for cr in crs: 211 if cr in results: 212 crs.remove(cr) 213 214 return results 215#################################################################### 216class ARCException(Exception): 217 """This covers arc.cgi script failure.""" 218 def __str__(self): 219 return "Error retrieving ARC data: %s" % (Exception.__str__(self)) 220 221def ARC(arclist, arcPath=None): 222 if not arcPath: 223 arcPath = "http://arc.opensolaris.org/cgi-bin/arc.cgi" 224 fields = ["present", "arc", "year", "case", "status", "title"] 225 opts = [("case", "%s/%s" % (a, c)) for a, c in arclist] 226 req = urllib2.Request(arcPath, urllib.urlencode(opts)) 227 try: 228 data = urllib2.urlopen(req).readlines() 229 except urllib2.HTTPError, e: 230 print "ERROR: HTTP error at " + req.get_full_url() + \ 231 " got error: " + str(e.code) 232 raise e 233 234 except urllib2.URLError, e: 235 print "ERROR: could not connect to " + req.get_full_url() + \ 236 ' got error: "' + e.reason[1] + '"' 237 raise e 238 ret = {} 239 for line in csv.DictReader(data, fields): 240 if line["present"] == "exists": 241 yc = "%s/%s" % (line["year"], line["case"]) 242 ret[(line["arc"], yc)] = line["title"] 243 elif line["present"] == "fatal": 244 raise ARCException(line["arc"]) 245 246 return ret 247 248#################################################################### 249 250# Pointers to the webrti server hostname & port to use 251# Using it directly is probably not *officially* supported, so we'll 252# have a pointer to the official `webrticli` command line interface 253# if using a direct socket connection fails for some reason, so we 254# have a fallback 255WEBRTI_HOST = 'webrti.sfbay.sun.com' 256WEBRTI_PORT = 9188 257WEBRTICLI = '/net/onnv.sfbay.sun.com/export/onnv-gate/public/bin/webrticli' 258 259 260class RtiException(Exception): 261 pass 262 263class RtiCallFailed(RtiException): 264 def __str__(self): 265 return "Unable to call webrti: %s" % (RtiException.__str__(self)) 266 267class RtiSystemProblem(RtiException): 268 def __str__(self): 269 return "RTI status cannot be determined for CR: %s" % (RtiException.__str__(self)) 270 271class RtiIncorrectCR(RtiException): 272 def __str__(self): 273 return "Incorrect CR number specified: %s" % (RtiException.__str__(self)) 274 275class RtiNotFound(RtiException): 276 def __str__(self): 277 return "RTI not found for CR: %s" % (RtiException.__str__(self)) 278 279class RtiNeedConsolidation(RtiException): 280 def __str__(self): 281 return "More than one consolidation has this CR: %s" % (RtiException.__str__(self)) 282 283class RtiBadGate(RtiException): 284 def __str__(self): 285 return "Incorrect gate name specified: %s" % (RtiException.__str__(self)) 286 287class RtiUnknownException(Exception): 288 def __str__(self): 289 return "Unknown webrti return code: %s" % (RtiException.__str__(self)) 290 291class RtiOffSwan(RtiException): 292 def __str__(self): 293 return "RTI status checks need SWAN access: %s" % (RtiException.__str__(self)) 294 295WEBRTI_ERRORS = { 296 '1': RtiSystemProblem, 297 '2': RtiIncorrectCR, 298 '3': RtiNotFound, 299 '4': RtiNeedConsolidation, 300 '5': RtiBadGate, 301} 302 303# Our Rti object which we'll use to represent an Rti query 304# It's really just a wrapper around the Rti connection, and attempts 305# to establish a direct socket connection and query the webrti server 306# directly (thus avoiding a system/fork/exec call). If it fails, it 307# falls back to the webrticli command line client. 308 309returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)') 310class Rti: 311 """Lookup an RTI. 312 313 Usage: 314 r = Rti("6640538") 315 print r.rtiNumber(); 316 """ 317 318 def __init__(self, cr, gate=None, consolidation=None): 319 """Create an Rti object for the specified change request. 320 321 Argument: 322 cr: change request id 323 324 Keyword arguments, to limit scope of RTI search: 325 gate: path to gate workspace (default=None) 326 consolidation: consolidation name (default=None) 327 """ 328 329 bufSz = 1024 330 addr = (WEBRTI_HOST, WEBRTI_PORT) 331 # If the passed 'cr' was given as an int, then wrap it 332 # into a string to make our life easier 333 if isinstance(cr, int): 334 cr = str(cr) 335 self.__queryCr = cr 336 self.__queryGate = gate 337 self.__queryConsolidation = consolidation 338 339 self.__webRtiOutput = [] 340 self.__mainCR = [] 341 self.__rtiNumber = [] 342 self.__consolidation = [] 343 self.__project = [] 344 self.__status = [] 345 self.__rtiType = [] 346 try: 347 # try to use a direct connection to the 348 # webrti server first 349 sock = socket(AF_INET, SOCK_STREAM) 350 sock.connect(addr) 351 command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr 352 if consolidation: 353 command += "-c\n%s\n" % consolidation 354 if gate: 355 command += "-g\n%s\n" % gate 356 command += "\n" 357 sock.send(command) 358 dataList = [] 359 # keep receiving data from the socket until the 360 # server closes the connection 361 stillReceiving = True 362 while stillReceiving: 363 dataPiece = sock.recv(bufSz) 364 if dataPiece: 365 dataList.append(dataPiece) 366 else: 367 stillReceiving = False 368 # create the lines, skipping the first 369 # ("WEBRTCLI/1.0\n") 370 data = '\n'.join(''.join(dataList).split('\n')[1:]) 371 except: 372 if not onSWAN(): 373 raise RtiOffSwan(cr) 374 375 if not os.path.exists(WEBRTICLI): 376 raise RtiCallFailed('not found') 377 378 # fallback to the "supported" webrticli interface 379 command = WEBRTICLI 380 if consolidation: 381 command += " -c " + consolidation 382 if gate: 383 command += " -g " + gate 384 command += " RTIstatus " + cr 385 386 try: 387 cliPipe = os.popen(command) 388 except: 389 # we couldn't call the webrticli for some 390 # reason, so return a failure 391 raise RtiCallFailed('unknown') 392 393 data = cliPipe.readline() 394 395 # parse the data to see if we got a return code 396 # if we did, then that's bad. if we didn't, 397 # then our call was successful 398 m = returnCodeRe.search(data) 399 if m: 400 rc = m.group(1) 401 # we got a return code, set it in our 402 # object, set the webRtiOutput for debugging 403 # or logging, and return a failure 404 if rc in WEBRTI_ERRORS: 405 exc = WEBRTI_ERRORS[rc] 406 if exc == RtiBadGate: 407 edata = gate 408 else: 409 edata = cr 410 else: 411 exc = RtiUnknownException 412 edata = rc 413 raise exc(edata) 414 415 data = data.splitlines() 416 # At this point, we should have valid data 417 for line in data: 418 line = line.rstrip('\r\n') 419 self.__webRtiOutput.append(line) 420 fields = line.split(':') 421 self.__mainCR.append(fields[0]) 422 self.__rtiNumber.append(fields[1]) 423 self.__consolidation.append(fields[2]) 424 self.__project.append(fields[3]) 425 self.__status.append(fields[4]) 426 self.__rtiType.append(fields[5]) 427 428 # accessors in case callers need the raw data 429 def mainCR(self): 430 return self.__mainCR 431 def rtiNumber(self): 432 return self.__rtiNumber 433 def consolidation(self): 434 return self.__consolidation 435 def project(self): 436 return self.__project 437 def status(self): 438 return self.__status 439 def rtiType(self): 440 return self.__rtiType 441 def queryCr(self): 442 return self.__queryCr 443 def queryGate(self): 444 return self.__queryGate 445 def queryConsolidation(self): 446 return self.__queryConsolidation 447 448 # in practice, most callers only care about the following 449 def accepted(self): 450 for status in self.__status: 451 if status != "S_ACCEPTED": 452 return False 453 return True 454 455 # for logging/debugging in case the caller wants the raw webrti output 456 def webRtiOutput(self): 457 return self.__webRtiOutput 458 459