1#! /usr/bin/python 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2008 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27 28# 29# Various database lookup classes/methods, i.e.: 30# * monaco 31# * bugs.opensolaris.org (b.o.o.) 32# * opensolaris.org/cgi/arc.py (for ARC) 33# 34 35import re 36import urllib 37import htmllib 38import os 39from socket import socket, AF_INET, SOCK_STREAM 40 41from onbld.Checks import onSWAN 42 43class BugException(Exception): 44 def __init__(self, data=''): 45 self.data = data 46 Exception.__init__(self, data) 47 48 def __str__(self): 49 return "Unknown error: %s" % self.data 50 51class NonExistentBug(BugException): 52 def __str__(self): 53 return "Bug %s does not exist" % self.data 54 55class Monaco(object): 56 """ 57 Query bug database. 58 59 Methods: 60 queryBugs() 61 expertQuery() 62 """ 63 64 def __init__(self): 65 self.__baseURL = "http://hestia.sfbay.sun.com/cgi-bin/expert?" 66 67 def expertQuery(self, cmd, format="Normal+text", header=False): 68 """Return results of user-supplied bug query. 69 70 Argument: 71 cmd: query to run 72 73 Keyword arguments: 74 format: desired output format (default="Normal+text") 75 header: include headers in output? (default=False) 76 77 Returns: 78 List of lines representing the output from Monaco 79 """ 80 81 url = self.__baseURL + "format=" + format + ";Go=2;" 82 if not header: url += "no_header=on;" 83 url += "cmds=" + urllib.quote_plus("\n".join(cmd)) 84 myMonaco = urllib.urlopen(url) 85 return myMonaco.readlines() 86 87 def queryBugs(self, crs): 88 """Return all info for requested change reports. 89 90 Argument: 91 crs: list of change request ids 92 93 Returns: 94 Dictionary, mapping CR=>dictionary, where the nested dictionary 95 is a mapping of field=>value 96 """ 97 monacoFields = [ "cr_number", "category", "sub_category", 98 "area", "release", "build", "responsible_manager", 99 "responsible_engineer", "priority", "status", "sub_status", 100 "submitted_by", "date_submitted", "synopsis" ] 101 cmd = [] 102 cmd.append("set What = cr." + ', cr.'.join(monacoFields)) 103 cmd.append("") 104 cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")") 105 cmd.append("") 106 cmd.append("set FinalClauses = order by cr.cr_number") 107 cmd.append("") 108 cmd.append("doMeta genQuery cr") 109 output = self.expertQuery(cmd, "Pipe-delimited+text") 110 results = {} 111 for line in output: 112 line = line.rstrip('\n') 113 114 # 115 # We request synopsis last, and split on only 116 # the number of separators that we expect to 117 # see such that a | in the synopsis doesn't 118 # throw us out of whack. 119 # 120 values = line.split('|', len(monacoFields) - 1) 121 v = 0 122 cr = values[0] 123 results[cr] = {} 124 for field in monacoFields: 125 results[cr][field] = values[v] 126 v += 1 127 return results 128 129class BooBug(object): 130 """Look up a single bug on bugs.opensolaris.org.""" 131 def __init__(self, cr): 132 cr = str(cr) 133 url = "http://bugs.opensolaris.org/view_bug.do?bug_id="+cr 134 data = urllib.urlopen(url).readlines() 135 self.__fields = {} 136 self.__fields["cr_number"] = cr 137 htmlParser = htmllib.HTMLParser(None) 138 metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$') 139 for line in data: 140 m = metaHtmlRe.search(line) 141 if not m: 142 continue 143 val = urllib.unquote(m.group(2)) 144 htmlParser.save_bgn() 145 htmlParser.feed(val) 146 self.__fields[m.group(1)] = htmlParser.save_end() 147 htmlParser.close() 148 if "synopsis" not in self.__fields: 149 raise NonExistentBug(cr) 150 151 def synopsis(self): 152 return self.__fields["synopsis"] 153 def product(self): 154 return self.__fields["product"] 155 def cat(self): 156 return self.__fields["category"] 157 def subcat(self): 158 return self.__fields["subcategory"] 159 def keywords(self): 160 return self.__fields["keywords"] 161 def state(self): 162 return self.__fields["state"] 163 def submit_date(self): 164 return self.__fields["submit_date"] 165 def type(self): 166 return self.__fields["type"] 167 def date(self): 168 return self.__fields["date"] 169 def number(self): 170 return self.__fields["cr_number"] 171 172class BugDB(object): 173 """Lookup change requests. 174 175 Object can be used on or off of SWAN, using either monaco or 176 bugs.opensolaris.org as a database. 177 178 Usage: 179 bdb = BugDB() 180 r = bdb.lookup("6455550") 181 print r["6455550"]["synopsis"] 182 r = bdb.lookup(["6455550", "6505625"]) 183 print r["6505625"]["synopsis"] 184 """ 185 186 def __init__(self, forceBoo = False): 187 """Create a BugDB object. 188 189 Keyword argument: 190 forceBoo: use b.o.o even from SWAN (default=False) 191 """ 192 if forceBoo: 193 self.__onSWAN = False 194 else: 195 self.__onSWAN = onSWAN() 196 if self.__onSWAN: 197 self.__m = Monaco() 198 199 def lookup(self, crs): 200 """Return all info for requested change reports. 201 202 Argument: 203 crs: one change request id (may be integer, string, or list), 204 or multiple change request ids (must be a list) 205 206 Returns: 207 Dictionary, mapping CR=>dictionary, where the nested dictionary 208 is a mapping of field=>value 209 """ 210 if not isinstance(crs, list): 211 crs = [str(crs)] 212 if self.__onSWAN: 213 results = self.__m.queryBugs(crs) 214 return self.__m.queryBugs(crs) 215 # else we're off-swan and querying via boo, which we can 216 # only do one bug at a time 217 results = {} 218 for cr in crs: 219 cr = str(cr) 220 try: 221 b = BooBug(cr) 222 except NonExistentBug: 223 continue 224 225 results[cr] = {} 226 results[cr]["cr_number"] = cr 227 results[cr]["product"] = b.product() 228 results[cr]["synopsis"] = b.synopsis() 229 results[cr]["category"] = b.cat() 230 results[cr]["sub_category"] = b.subcat() 231 results[cr]["keywords"] = b.keywords() 232 results[cr]["status"] = b.state() 233 results[cr]["date_submitted"] = b.submit_date() 234 results[cr]["type"] = b.type() 235 results[cr]["date"] = b.date() 236 237 return results 238 239#################################################################### 240 241class ARC(object): 242 """Lookup an ARC case on opensolaris.org. 243 244 Usage: 245 a = ARC("PSARC", "2008/002") 246 if a.valid(): 247 print a.name() 248 """ 249 def __init__(self, arc, case): 250 self.__valid = False 251 q = "http://opensolaris.org/cgi/arc.py?n=1" 252 q += "&arc0=" + arc 253 q += "&case0=" + case 254 data = urllib.urlopen(q).readlines() 255 self.__fields = {} 256 for line in data: 257 line = line.rstrip('\n') 258 fields = line.split('|') 259 validity = fields[0] 260 261 if validity != "0": 262 return 263 else: 264 self.__fields["Name"] = fields[2] 265 266 self.__valid = True 267 268 def valid(self): 269 return self.__valid 270 def name(self): 271 return self.__fields["Name"] 272 def status(self): 273 return self.__fields["Status"] 274 def type(self): 275 return self.__fields["Type"] 276 277#################################################################### 278 279# Pointers to the webrti server hostname & port to use 280# Using it directly is probably not *officially* supported, so we'll 281# have a pointer to the official `webrticli` command line interface 282# if using a direct socket connection fails for some reason, so we 283# have a fallback 284WEBRTI_HOST = 'webrti.sfbay.sun.com' 285WEBRTI_PORT = 9188 286WEBRTICLI = '/net/webrti.sfbay.sun.com/export/home/bin/webrticli' 287 288 289class RtiException(Exception): 290 def __init__(self, data=''): 291 self.data = data 292 Exception.__init__(self, data) 293 294 def __str__(self): 295 return "Unknown error: %s" % self.data 296 297# RtiInvalidOutput & RtiCallFailed are our "own" failures 298# The other exceptions are triggered from WebRTI itself 299class RtiInvalidOutput(RtiException): 300 def __str__(self): 301 return "Invalid output from WebRTI: %s" % self.data 302 303class RtiCallFailed(RtiException): 304 def __str__(self): 305 return "Unable to call webrti: %s" % self.data 306 307class RtiSystemProblem(RtiException): 308 def __str__(self): 309 return "RTI status cannot be determined: %s" % self.data 310 311class RtiIncorrectCR(RtiException): 312 def __str__(self): 313 return "Incorrect CR number specified: %s" % self.data 314 315class RtiNotFound(RtiException): 316 def __str__(self): 317 return "RTI not found: %s" % self.data 318 319class RtiNeedConsolidation(RtiException): 320 def __str__(self): 321 return "More than one consolidation has this CR: %s" % self.data 322 323class RtiBadGate(RtiException): 324 def __str__(self): 325 return "Incorrect gate name specified: %s" % self.data 326 327class RtiOffSwan(RtiException): 328 def __str__(self): 329 return "RTI status checks need SWAN access: %s" % self.data 330 331WEBRTI_ERRORS = { 332 '1': RtiSystemProblem, 333 '2': RtiIncorrectCR, 334 '3': RtiNotFound, 335 '4': RtiNeedConsolidation, 336 '5': RtiBadGate, 337} 338 339# Our Rti object which we'll use to represent an Rti query 340# It's really just a wrapper around the Rti connection, and attempts 341# to establish a direct socket connection and query the webrti server 342# directly (thus avoiding a system/fork/exec call). If it fails, it 343# falls back to the webrticli command line client. 344 345returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)') 346class Rti: 347 """Lookup an RTI. 348 349 Usage: 350 r = Rti("6640538") 351 print r.rtiNumber(); 352 """ 353 354 def __init__(self, cr, gate=None, consolidation=None): 355 """Create an Rti object for the specified change request. 356 357 Argument: 358 cr: change request id 359 360 Keyword arguments, to limit scope of RTI search: 361 gate: path to gate workspace (default=None) 362 consolidation: consolidation name (default=None) 363 """ 364 365 bufSz = 1024 366 addr = (WEBRTI_HOST, WEBRTI_PORT) 367 # If the passed 'cr' was given as an int, then wrap it 368 # into a string to make our life easier 369 if isinstance(cr, int): 370 cr = str(cr) 371 self.__queryCr = cr 372 self.__queryGate = gate 373 self.__queryConsolidation = consolidation 374 375 try: 376 # try to use a direct connection to the 377 # webrti server first 378 sock = socket(AF_INET, SOCK_STREAM) 379 sock.connect(addr) 380 command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr 381 if consolidation: 382 command += "-c\n%s\n" % consolidation 383 if gate: 384 command += "-g\n%s\n" % gate 385 command += "\n" 386 sock.send(command) 387 dataList = [] 388 # keep receiving data from the socket until the 389 # server closes the connection 390 stillReceiving = True 391 while stillReceiving: 392 dataPiece = sock.recv(bufSz) 393 if dataPiece: 394 dataList.append(dataPiece) 395 else: 396 stillReceiving = False 397 # create the lines, skipping the first 398 # ("WEBRTCLI/1.0\n") 399 data = '\n'.join(''.join(dataList).split('\n')[1:]) 400 except: 401 if not onSWAN(): 402 raise RtiOffSwan(cr) 403 404 if not os.path.exists(WEBRTICLI): 405 raise RtiCallFailed('not found') 406 407 # fallback to the "supported" webrticli interface 408 command = WEBRTICLI 409 if consolidation: 410 command += " -c " + consolidation 411 if gate: 412 command += " -g " + gate 413 command += " RTIstatus " + cr 414 415 try: 416 cliPipe = os.popen(command) 417 except: 418 # we couldn't call the webrticli for some 419 # reason, so return a failure 420 raise RtiCallFailed('unknown') 421 422 data = cliPipe.readline() 423 424 # parse the data to see if we got a return code 425 # if we did, then that's bad. if we didn't, 426 # then our call was successfully 427 m = returnCodeRe.search(data) 428 if m: 429 # we got a return code, set it in our 430 # object, set the webRtiOutput for debugging 431 # or logging, and return a failure 432 if m.group(1) in WEBRTI_ERRORS: 433 exc = WEBRTI_ERRORS[m.group(1)] 434 else: 435 exc = RtiException 436 raise exc(data) 437 438 if data.count('\n') != 1: 439 # there shouldn't be more than one line in 440 # the output. if we got more than one line, 441 # then let's be paranoid, and abort. 442 raise RtiInvalidOutput(data) 443 444 # At this point, we should have valid data 445 data = data.rstrip('\r\n') 446 self.__webRtiOutput = data 447 self.__fields = data.split(':') 448 self.__mainCR = self.__fields[0] 449 self.__rtiNumber = self.__fields[1] 450 self.__consolidation = self.__fields[2] 451 self.__project = self.__fields[3] 452 self.__status = self.__fields[4] 453 self.__rtiType = self.__fields[5] 454 455 # accessors in case callers need the raw data 456 def mainCR(self): 457 return self.__mainCR 458 def rtiNumber(self): 459 return self.__rtiNumber 460 def consolidation(self): 461 return self.__consolidation 462 def project(self): 463 return self.__project 464 def status(self): 465 return self.__status 466 def rtiType(self): 467 return self.__rtiType 468 def queryCr(self): 469 return self.__queryCr 470 def queryGate(self): 471 return self.__queryGate 472 def queryConsolidation(self): 473 return self.__queryConsolidation 474 475 # in practice, most callers only care about the following 476 def accepted(self): 477 return (self.__status == "S_ACCEPTED") 478 479 # for logging/debugging in case the caller wants the raw webrti output 480 def webRtiOutput(self): 481 return self.__webRtiOutput 482 483 484