1#! /usr/bin/python 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2008 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27# ident "%Z%%M% %I% %E% SMI" 28# 29 30# 31# Various database lookup classes/methods, i.e.: 32# * monaco 33# * bugs.opensolaris.org (b.o.o.) 34# * opensolaris.org/cgi/arc.py (for ARC) 35# 36 37import re 38import urllib 39import htmllib 40import os 41from socket import socket, AF_INET, SOCK_STREAM 42 43from onbld.Checks import onSWAN 44 45class BugException(Exception): 46 def __init__(self, data=''): 47 self.data = data 48 Exception.__init__(self, data) 49 50 def __str__(self): 51 return "Unknown error: %s" % self.data 52 53class NonExistentBug(BugException): 54 def __str__(self): 55 return "Bug %s does not exist" % self.data 56 57class Monaco(object): 58 """ 59 Query bug database. 60 61 Methods: 62 queryBugs() 63 expertQuery() 64 """ 65 66 def __init__(self): 67 self.__baseURL = "http://hestia.sfbay.sun.com/cgi-bin/expert?" 68 69 def expertQuery(self, cmd, format="Normal+text", header=False): 70 """Return results of user-supplied bug query. 71 72 Argument: 73 cmd: query to run 74 75 Keyword arguments: 76 format: desired output format (default="Normal+text") 77 header: include headers in output? (default=False) 78 79 Returns: 80 List of lines representing the output from Monaco 81 """ 82 83 url = self.__baseURL + "format=" + format + ";Go=2;" 84 if not header: url += "no_header=on;" 85 url += "cmds=" + urllib.quote_plus("\n".join(cmd)) 86 myMonaco = urllib.urlopen(url) 87 return myMonaco.readlines() 88 89 def queryBugs(self, crs): 90 """Return all info for requested change reports. 91 92 Argument: 93 crs: list of change request ids 94 95 Returns: 96 Dictionary, mapping CR=>dictionary, where the nested dictionary 97 is a mapping of field=>value 98 """ 99 monacoFields = [ "cr_number", "synopsis", "category", "sub_category", 100 "area", "release", "build", "responsible_manager", 101 "responsible_engineer", "priority", "status", "sub_status", 102 "submitted_by", "date_submitted" ] 103 cmd = [] 104 cmd.append("set What = cr." + ', cr.'.join(monacoFields)) 105 cmd.append("") 106 cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")") 107 cmd.append("") 108 cmd.append("set FinalClauses = order by cr.cr_number") 109 cmd.append("") 110 cmd.append("doMeta genQuery cr") 111 output = self.expertQuery(cmd, "Pipe-delimited+text") 112 results = {} 113 for line in output: 114 values = line.split('|') 115 v = 0 116 cr = values[0] 117 results[cr] = {} 118 for field in monacoFields: 119 results[cr][field] = values[v] 120 v += 1 121 return results 122 123class BooBug(object): 124 """Look up a single bug on bugs.opensolaris.org.""" 125 def __init__(self, cr): 126 cr = str(cr) 127 url = "http://bugs.opensolaris.org/view_bug.do?bug_id="+cr 128 data = urllib.urlopen(url).readlines() 129 self.__fields = {} 130 self.__fields["cr_number"] = cr 131 htmlParser = htmllib.HTMLParser(None) 132 metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$') 133 for line in data: 134 m = metaHtmlRe.search(line) 135 if not m: 136 continue 137 val = urllib.unquote(m.group(2)) 138 htmlParser.save_bgn() 139 htmlParser.feed(val) 140 self.__fields[m.group(1)] = htmlParser.save_end() 141 htmlParser.close() 142 if "synopsis" not in self.__fields: 143 raise NonExistentBug(cr) 144 145 def synopsis(self): 146 return self.__fields["synopsis"] 147 def product(self): 148 return self.__fields["product"] 149 def cat(self): 150 return self.__fields["category"] 151 def subcat(self): 152 return self.__fields["subcategory"] 153 def keywords(self): 154 return self.__fields["keywords"] 155 def state(self): 156 return self.__fields["state"] 157 def submit_date(self): 158 return self.__fields["submit_date"] 159 def type(self): 160 return self.__fields["type"] 161 def date(self): 162 return self.__fields["date"] 163 def number(self): 164 return self.__fields["cr_number"] 165 166class BugDB(object): 167 """Lookup change requests. 168 169 Object can be used on or off of SWAN, using either monaco or 170 bugs.opensolaris.org as a database. 171 172 Usage: 173 bdb = BugDB() 174 r = bdb.lookup("6455550") 175 print r["6455550"]["synopsis"] 176 r = bdb.lookup(["6455550", "6505625"]) 177 print r["6505625"]["synopsis"] 178 """ 179 180 def __init__(self, forceBoo = False): 181 """Create a BugDB object. 182 183 Keyword argument: 184 forceBoo: use b.o.o even from SWAN (default=False) 185 """ 186 if forceBoo: 187 self.__onSWAN = False 188 else: 189 self.__onSWAN = onSWAN() 190 if self.__onSWAN: 191 self.__m = Monaco() 192 193 def lookup(self, crs): 194 """Return all info for requested change reports. 195 196 Argument: 197 crs: one change request id (may be integer, string, or list), 198 or multiple change request ids (must be a list) 199 200 Returns: 201 Dictionary, mapping CR=>dictionary, where the nested dictionary 202 is a mapping of field=>value 203 """ 204 if not isinstance(crs, list): 205 crs = [str(crs)] 206 if self.__onSWAN: 207 results = self.__m.queryBugs(crs) 208 return self.__m.queryBugs(crs) 209 # else we're off-swan and querying via boo, which we can 210 # only do one bug at a time 211 results = {} 212 for cr in crs: 213 cr = str(cr) 214 try: 215 b = BooBug(cr) 216 except NonExistentBug: 217 continue 218 219 results[cr] = {} 220 results[cr]["cr_number"] = cr 221 results[cr]["product"] = b.product() 222 results[cr]["synopsis"] = b.synopsis() 223 results[cr]["category"] = b.cat() 224 results[cr]["sub_category"] = b.subcat() 225 results[cr]["keywords"] = b.keywords() 226 results[cr]["status"] = b.state() 227 results[cr]["date_submitted"] = b.submit_date() 228 results[cr]["type"] = b.type() 229 results[cr]["date"] = b.date() 230 231 return results 232 233#################################################################### 234 235class ARC(object): 236 """Lookup an ARC case on opensolaris.org. 237 238 Usage: 239 a = ARC("PSARC", "2008/002") 240 if a.valid(): 241 print a.name() 242 """ 243 def __init__(self, arc, case): 244 self.__valid = False 245 q = "http://opensolaris.org/cgi/arc.py?n=1" 246 q += "&arc0=" + arc 247 q += "&case0=" + case 248 data = urllib.urlopen(q).readlines() 249 self.__fields = {} 250 for line in data: 251 line = line.rstrip('\n') 252 fields = line.split('|') 253 validity = fields[0] 254 255 if validity != "0": 256 return 257 else: 258 self.__fields["Name"] = fields[2] 259 260 self.__valid = True 261 262 def valid(self): 263 return self.__valid 264 def name(self): 265 return self.__fields["Name"] 266 def status(self): 267 return self.__fields["Status"] 268 def type(self): 269 return self.__fields["Type"] 270 271#################################################################### 272 273# Pointers to the webrti server hostname & port to use 274# Using it directly is probably not *officially* supported, so we'll 275# have a pointer to the official `webrticli` command line interface 276# if using a direct socket connection fails for some reason, so we 277# have a fallback 278WEBRTI_HOST = 'webrti.sfbay.sun.com' 279WEBRTI_PORT = 9188 280WEBRTICLI = '/net/webrti.sfbay.sun.com/export/home/bin/webrticli' 281 282 283class RtiException(Exception): 284 def __init__(self, data=''): 285 self.data = data 286 Exception.__init__(self, data) 287 288 def __str__(self): 289 return "Unknown error: %s" % self.data 290 291# RtiInvalidOutput & RtiCallFailed are our "own" failures 292# The other exceptions are triggered from WebRTI itself 293class RtiInvalidOutput(RtiException): 294 def __str__(self): 295 return "Invalid output from WebRTI: %s" % self.data 296 297class RtiCallFailed(RtiException): 298 def __str__(self): 299 return "Unable to call webrti: %s" % self.data 300 301class RtiSystemProblem(RtiException): 302 def __str__(self): 303 return "RTI status cannot be determined: %s" % self.data 304 305class RtiIncorrectCR(RtiException): 306 def __str__(self): 307 return "Incorrect CR number specified: %s" % self.data 308 309class RtiNotFound(RtiException): 310 def __str__(self): 311 return "RTI not found: %s" % self.data 312 313class RtiNeedConsolidation(RtiException): 314 def __str__(self): 315 return "More than one consolidation has this CR: %s" % self.data 316 317class RtiBadGate(RtiException): 318 def __str__(self): 319 return "Incorrect gate name specified: %s" % self.data 320 321class RtiOffSwan(RtiException): 322 def __str__(self): 323 return "RTI status checks need SWAN access: %s" % self.data 324 325WEBRTI_ERRORS = { 326 '1': RtiSystemProblem, 327 '2': RtiIncorrectCR, 328 '3': RtiNotFound, 329 '4': RtiNeedConsolidation, 330 '5': RtiBadGate, 331} 332 333# Our Rti object which we'll use to represent an Rti query 334# It's really just a wrapper around the Rti connection, and attempts 335# to establish a direct socket connection and query the webrti server 336# directly (thus avoiding a system/fork/exec call). If it fails, it 337# falls back to the webrticli command line client. 338 339returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)') 340class Rti: 341 """Lookup an RTI. 342 343 Usage: 344 r = Rti("6640538") 345 print r.rtiNumber(); 346 """ 347 348 def __init__(self, cr, gate=None, consolidation=None): 349 """Create an Rti object for the specified change request. 350 351 Argument: 352 cr: change request id 353 354 Keyword arguments, to limit scope of RTI search: 355 gate: path to gate workspace (default=None) 356 consolidation: consolidation name (default=None) 357 """ 358 359 bufSz = 1024 360 addr = (WEBRTI_HOST, WEBRTI_PORT) 361 # If the passed 'cr' was given as an int, then wrap it 362 # into a string to make our life easier 363 if isinstance(cr, int): 364 cr = str(cr) 365 self.__queryCr = cr 366 self.__queryGate = gate 367 self.__queryConsolidation = consolidation 368 369 try: 370 # try to use a direct connection to the 371 # webrti server first 372 sock = socket(AF_INET, SOCK_STREAM) 373 sock.connect(addr) 374 command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr 375 if consolidation: 376 command += "-c\n%s\n" % consolidation 377 if gate: 378 command += "-g\n%s\n" % gate 379 command += "\n" 380 sock.send(command) 381 dataList = [] 382 # keep receiving data from the socket until the 383 # server closes the connection 384 stillReceiving = True 385 while stillReceiving: 386 dataPiece = sock.recv(bufSz) 387 if dataPiece: 388 dataList.append(dataPiece) 389 else: 390 stillReceiving = False 391 # create the lines, skipping the first 392 # ("WEBRTCLI/1.0\n") 393 data = '\n'.join(''.join(dataList).split('\n')[1:]) 394 except: 395 if not onSWAN(): 396 raise RtiOffSwan(cr) 397 398 if not os.path.exists(WEBRTICLI): 399 raise RtiCallFailed('not found') 400 401 # fallback to the "supported" webrticli interface 402 command = WEBRTICLI 403 if consolidation: 404 command += " -c " + consolidation 405 if gate: 406 command += " -g " + gate 407 command += " RTIstatus " + cr 408 409 try: 410 cliPipe = os.popen(command) 411 except: 412 # we couldn't call the webrticli for some 413 # reason, so return a failure 414 raise RtiCallFailed('unknown') 415 416 data = cliPipe.readline() 417 418 # parse the data to see if we got a return code 419 # if we did, then that's bad. if we didn't, 420 # then our call was successfully 421 m = returnCodeRe.search(data) 422 if m: 423 # we got a return code, set it in our 424 # object, set the webRtiOutput for debugging 425 # or logging, and return a failure 426 if m.group(1) in WEBRTI_ERRORS: 427 exc = WEBRTI_ERRORS[m.group(1)] 428 else: 429 exc = RtiException 430 raise exc(data) 431 432 if data.count('\n') != 1: 433 # there shouldn't be more than one line in 434 # the output. if we got more than one line, 435 # then let's be paranoid, and abort. 436 raise RtiInvalidOutput(data) 437 438 # At this point, we should have valid data 439 data = data.rstrip('\r\n') 440 self.__webRtiOutput = data 441 self.__fields = data.split(':') 442 self.__mainCR = self.__fields[0] 443 self.__rtiNumber = self.__fields[1] 444 self.__consolidation = self.__fields[2] 445 self.__project = self.__fields[3] 446 self.__status = self.__fields[4] 447 self.__rtiType = self.__fields[5] 448 449 # accessors in case callers need the raw data 450 def mainCR(self): 451 return self.__mainCR 452 def rtiNumber(self): 453 return self.__rtiNumber 454 def consolidation(self): 455 return self.__consolidation 456 def project(self): 457 return self.__project 458 def status(self): 459 return self.__status 460 def rtiType(self): 461 return self.__rtiType 462 def queryCr(self): 463 return self.__queryCr 464 def queryGate(self): 465 return self.__queryGate 466 def queryConsolidation(self): 467 return self.__queryConsolidation 468 469 # in practice, most callers only care about the following 470 def accepted(self): 471 return (self.__status == "S_ACCEPTED") 472 473 # for logging/debugging in case the caller wants the raw webrti output 474 def webRtiOutput(self): 475 return self.__webRtiOutput 476 477 478