1#! /usr/bin/python 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2008 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27 28# 29# Various database lookup classes/methods, i.e.: 30# * monaco 31# * bugs.opensolaris.org (b.o.o.) 32# * opensolaris.org/cgi/arc.py (for ARC) 33# 34 35import re 36import urllib 37import htmllib 38import os 39from socket import socket, AF_INET, SOCK_STREAM 40 41from onbld.Checks import onSWAN 42 43class BugException(Exception): 44 def __init__(self, data=''): 45 self.data = data 46 Exception.__init__(self, data) 47 48 def __str__(self): 49 return "Unknown error: %s" % self.data 50 51class NonExistentBug(BugException): 52 def __str__(self): 53 return "Bug %s does not exist" % self.data 54 55class Monaco(object): 56 """ 57 Query bug database. 58 59 Methods: 60 queryBugs() 61 expertQuery() 62 """ 63 64 def __init__(self): 65 self.__baseURL = "http://hestia.sfbay.sun.com/cgi-bin/expert?" 66 67 def expertQuery(self, cmd, format="Normal+text", header=False): 68 """Return results of user-supplied bug query. 69 70 Argument: 71 cmd: query to run 72 73 Keyword arguments: 74 format: desired output format (default="Normal+text") 75 header: include headers in output? (default=False) 76 77 Returns: 78 List of lines representing the output from Monaco 79 """ 80 81 url = self.__baseURL + "format=" + format + ";Go=2;" 82 if not header: url += "no_header=on;" 83 url += "cmds=" + urllib.quote_plus("\n".join(cmd)) 84 myMonaco = urllib.urlopen(url) 85 return myMonaco.readlines() 86 87 def queryBugs(self, crs): 88 """Return all info for requested change reports. 89 90 Argument: 91 crs: list of change request ids 92 93 Returns: 94 Dictionary, mapping CR=>dictionary, where the nested dictionary 95 is a mapping of field=>value 96 """ 97 monacoFields = [ "cr_number", "category", "sub_category", 98 "area", "release", "build", "responsible_manager", 99 "responsible_engineer", "priority", "status", "sub_status", 100 "submitted_by", "date_submitted", "synopsis" ] 101 cmd = [] 102 cmd.append("set What = cr." + ', cr.'.join(monacoFields)) 103 cmd.append("") 104 cmd.append("set Which = cr.cr_number in (" + ','.join(crs) +")") 105 cmd.append("") 106 cmd.append("set FinalClauses = order by cr.cr_number") 107 cmd.append("") 108 cmd.append("doMeta genQuery cr") 109 output = self.expertQuery(cmd, "Pipe-delimited+text") 110 results = {} 111 for line in output: 112 line = line.rstrip('\n') 113 114 # 115 # We request synopsis last, and split on only 116 # the number of separators that we expect to 117 # see such that a | in the synopsis doesn't 118 # throw us out of whack. 119 # 120 values = line.split('|', len(monacoFields) - 1) 121 v = 0 122 cr = values[0] 123 results[cr] = {} 124 for field in monacoFields: 125 results[cr][field] = values[v] 126 v += 1 127 return results 128 129class BooBug(object): 130 """Look up a single bug on bugs.opensolaris.org.""" 131 def __init__(self, cr): 132 cr = str(cr) 133 url = "http://bugs.opensolaris.org/view_bug.do?bug_id="+cr 134 data = urllib.urlopen(url).readlines() 135 self.__fields = {} 136 self.__fields["cr_number"] = cr 137 htmlParser = htmllib.HTMLParser(None) 138 metaHtmlRe = re.compile(r'^<meta name="([^"]+)" content="([^"]*)">$') 139 for line in data: 140 m = metaHtmlRe.search(line) 141 if not m: 142 continue 143 val = urllib.unquote(m.group(2)) 144 htmlParser.save_bgn() 145 htmlParser.feed(val) 146 self.__fields[m.group(1)] = htmlParser.save_end() 147 htmlParser.close() 148 if "synopsis" not in self.__fields: 149 raise NonExistentBug(cr) 150 151 def synopsis(self): 152 return self.__fields["synopsis"] 153 def product(self): 154 return self.__fields["product"] 155 def cat(self): 156 return self.__fields["category"] 157 def subcat(self): 158 return self.__fields["subcategory"] 159 def keywords(self): 160 return self.__fields["keywords"] 161 def state(self): 162 return self.__fields["state"] 163 def submit_date(self): 164 return self.__fields["submit_date"] 165 def type(self): 166 return self.__fields["type"] 167 def date(self): 168 return self.__fields["date"] 169 def number(self): 170 return self.__fields["cr_number"] 171 172class BugDB(object): 173 """Lookup change requests. 174 175 Object can be used on or off of SWAN, using either monaco or 176 bugs.opensolaris.org as a database. 177 178 Usage: 179 bdb = BugDB() 180 r = bdb.lookup("6455550") 181 print r["6455550"]["synopsis"] 182 r = bdb.lookup(["6455550", "6505625"]) 183 print r["6505625"]["synopsis"] 184 """ 185 186 def __init__(self, forceBoo = False): 187 """Create a BugDB object. 188 189 Keyword argument: 190 forceBoo: use b.o.o even from SWAN (default=False) 191 """ 192 if forceBoo: 193 self.__onSWAN = False 194 else: 195 self.__onSWAN = onSWAN() 196 if self.__onSWAN: 197 self.__m = Monaco() 198 199 def lookup(self, crs): 200 """Return all info for requested change reports. 201 202 Argument: 203 crs: one change request id (may be integer, string, or list), 204 or multiple change request ids (must be a list) 205 206 Returns: 207 Dictionary, mapping CR=>dictionary, where the nested dictionary 208 is a mapping of field=>value 209 """ 210 if not isinstance(crs, list): 211 crs = [str(crs)] 212 if self.__onSWAN: 213 results = self.__m.queryBugs(crs) 214 return self.__m.queryBugs(crs) 215 # else we're off-swan and querying via boo, which we can 216 # only do one bug at a time 217 results = {} 218 for cr in crs: 219 cr = str(cr) 220 try: 221 b = BooBug(cr) 222 except NonExistentBug: 223 continue 224 225 results[cr] = {} 226 results[cr]["cr_number"] = cr 227 results[cr]["product"] = b.product() 228 results[cr]["synopsis"] = b.synopsis() 229 results[cr]["category"] = b.cat() 230 results[cr]["sub_category"] = b.subcat() 231 results[cr]["keywords"] = b.keywords() 232 results[cr]["status"] = b.state() 233 results[cr]["date_submitted"] = b.submit_date() 234 results[cr]["type"] = b.type() 235 results[cr]["date"] = b.date() 236 237 return results 238 239#################################################################### 240 241class ARC(object): 242 """Lookup an ARC case on opensolaris.org. 243 244 Usage: 245 a = ARC("PSARC", "2008/002") 246 if a.valid(): 247 print a.name() 248 """ 249 def __init__(self, arc, case): 250 self.__valid = False 251 q = "http://opensolaris.org/cgi/arc.py?n=1" 252 q += "&arc0=" + arc 253 q += "&case0=" + case 254 data = urllib.urlopen(q).readlines() 255 self.__fields = {} 256 for line in data: 257 line = line.rstrip('\n') 258 fields = line.split('|') 259 validity = fields[0] 260 261 if validity != "0": 262 return 263 else: 264 self.__fields["Name"] = fields[2] 265 266 self.__valid = True 267 268 def valid(self): 269 return self.__valid 270 def name(self): 271 return self.__fields["Name"] 272 def status(self): 273 return self.__fields["Status"] 274 def type(self): 275 return self.__fields["Type"] 276 277#################################################################### 278 279# Pointers to the webrti server hostname & port to use 280# Using it directly is probably not *officially* supported, so we'll 281# have a pointer to the official `webrticli` command line interface 282# if using a direct socket connection fails for some reason, so we 283# have a fallback 284WEBRTI_HOST = 'webrti.sfbay.sun.com' 285WEBRTI_PORT = 9188 286WEBRTICLI = '/net/onnv.sfbay.sun.com/export/onnv-gate/public/bin/webrticli' 287 288 289class RtiException(Exception): 290 def __init__(self, data=''): 291 self.data = data 292 Exception.__init__(self, data) 293 294 def __str__(self): 295 return "Unknown error: %s" % self.data 296 297class RtiCallFailed(RtiException): 298 def __str__(self): 299 return "Unable to call webrti: %s" % self.data 300 301class RtiSystemProblem(RtiException): 302 def __str__(self): 303 return "RTI status cannot be determined: %s" % self.data 304 305class RtiIncorrectCR(RtiException): 306 def __str__(self): 307 return "Incorrect CR number specified: %s" % self.data 308 309class RtiNotFound(RtiException): 310 def __str__(self): 311 return "RTI not found: %s" % self.data 312 313class RtiNeedConsolidation(RtiException): 314 def __str__(self): 315 return "More than one consolidation has this CR: %s" % self.data 316 317class RtiBadGate(RtiException): 318 def __str__(self): 319 return "Incorrect gate name specified: %s" % self.data 320 321class RtiOffSwan(RtiException): 322 def __str__(self): 323 return "RTI status checks need SWAN access: %s" % self.data 324 325WEBRTI_ERRORS = { 326 '1': RtiSystemProblem, 327 '2': RtiIncorrectCR, 328 '3': RtiNotFound, 329 '4': RtiNeedConsolidation, 330 '5': RtiBadGate, 331} 332 333# Our Rti object which we'll use to represent an Rti query 334# It's really just a wrapper around the Rti connection, and attempts 335# to establish a direct socket connection and query the webrti server 336# directly (thus avoiding a system/fork/exec call). If it fails, it 337# falls back to the webrticli command line client. 338 339returnCodeRe = re.compile(r'.*RETURN_CODE=(\d+)') 340class Rti: 341 """Lookup an RTI. 342 343 Usage: 344 r = Rti("6640538") 345 print r.rtiNumber(); 346 """ 347 348 def __init__(self, cr, gate=None, consolidation=None): 349 """Create an Rti object for the specified change request. 350 351 Argument: 352 cr: change request id 353 354 Keyword arguments, to limit scope of RTI search: 355 gate: path to gate workspace (default=None) 356 consolidation: consolidation name (default=None) 357 """ 358 359 bufSz = 1024 360 addr = (WEBRTI_HOST, WEBRTI_PORT) 361 # If the passed 'cr' was given as an int, then wrap it 362 # into a string to make our life easier 363 if isinstance(cr, int): 364 cr = str(cr) 365 self.__queryCr = cr 366 self.__queryGate = gate 367 self.__queryConsolidation = consolidation 368 369 self.__webRtiOutput = [] 370 self.__mainCR = [] 371 self.__rtiNumber = [] 372 self.__consolidation = [] 373 self.__project = [] 374 self.__status = [] 375 self.__rtiType = [] 376 try: 377 # try to use a direct connection to the 378 # webrti server first 379 sock = socket(AF_INET, SOCK_STREAM) 380 sock.connect(addr) 381 command = "WEBRTICLI/1.0\nRTIstatus\n%s\n" % cr 382 if consolidation: 383 command += "-c\n%s\n" % consolidation 384 if gate: 385 command += "-g\n%s\n" % gate 386 command += "\n" 387 sock.send(command) 388 dataList = [] 389 # keep receiving data from the socket until the 390 # server closes the connection 391 stillReceiving = True 392 while stillReceiving: 393 dataPiece = sock.recv(bufSz) 394 if dataPiece: 395 dataList.append(dataPiece) 396 else: 397 stillReceiving = False 398 # create the lines, skipping the first 399 # ("WEBRTCLI/1.0\n") 400 data = '\n'.join(''.join(dataList).split('\n')[1:]) 401 except: 402 if not onSWAN(): 403 raise RtiOffSwan(cr) 404 405 if not os.path.exists(WEBRTICLI): 406 raise RtiCallFailed('not found') 407 408 # fallback to the "supported" webrticli interface 409 command = WEBRTICLI 410 if consolidation: 411 command += " -c " + consolidation 412 if gate: 413 command += " -g " + gate 414 command += " RTIstatus " + cr 415 416 try: 417 cliPipe = os.popen(command) 418 except: 419 # we couldn't call the webrticli for some 420 # reason, so return a failure 421 raise RtiCallFailed('unknown') 422 423 data = cliPipe.readline() 424 425 # parse the data to see if we got a return code 426 # if we did, then that's bad. if we didn't, 427 # then our call was successfully 428 m = returnCodeRe.search(data) 429 if m: 430 # we got a return code, set it in our 431 # object, set the webRtiOutput for debugging 432 # or logging, and return a failure 433 if m.group(1) in WEBRTI_ERRORS: 434 exc = WEBRTI_ERRORS[m.group(1)] 435 else: 436 exc = RtiException 437 raise exc(data) 438 439 data = data.splitlines() 440 # At this point, we should have valid data 441 for line in data: 442 line = line.rstrip('\r\n') 443 self.__webRtiOutput.append(line) 444 fields = line.split(':') 445 self.__mainCR.append(fields[0]) 446 self.__rtiNumber.append(fields[1]) 447 self.__consolidation.append(fields[2]) 448 self.__project.append(fields[3]) 449 self.__status.append(fields[4]) 450 self.__rtiType.append(fields[5]) 451 452 # accessors in case callers need the raw data 453 def mainCR(self): 454 return self.__mainCR 455 def rtiNumber(self): 456 return self.__rtiNumber 457 def consolidation(self): 458 return self.__consolidation 459 def project(self): 460 return self.__project 461 def status(self): 462 return self.__status 463 def rtiType(self): 464 return self.__rtiType 465 def queryCr(self): 466 return self.__queryCr 467 def queryGate(self): 468 return self.__queryGate 469 def queryConsolidation(self): 470 return self.__queryConsolidation 471 472 # in practice, most callers only care about the following 473 def accepted(self): 474 for status in self.__status: 475 if status != "S_ACCEPTED": 476 return False 477 return True 478 479 # for logging/debugging in case the caller wants the raw webrti output 480 def webRtiOutput(self): 481 return self.__webRtiOutput 482 483