1''' 2 Copyright 2011 by the Massachusetts 3 Institute of Technology. All Rights Reserved. 4 5 Export of this software from the United States of America may 6 require a specific license from the United States Government. 7 It is the responsibility of any person or organization contemplating 8 export to obtain such a license before exporting. 9 10 WITHIN THAT CONSTRAINT, permission to use, copy, modify, and 11 distribute this software and its documentation for any purpose and 12 without fee is hereby granted, provided that the above copyright 13 notice appear in all copies and that both that copyright notice and 14 this permission notice appear in supporting documentation, and that 15 the name of M.I.T. not be used in advertising or publicity pertaining 16 to distribution of the software without specific, written prior 17 permission. Furthermore if you modify this software you must label 18 your software as modified software and not distribute it in such a 19 fashion that it might be confused with the original M.I.T. software. 20 M.I.T. makes no representations about the suitability of 21 this software for any purpose. It is provided "as is" without express 22 or implied warranty. 23''' 24import sys 25import re 26 27from collections import defaultdict 28from xml.sax import make_parser 29from xml.sax.handler import ContentHandler 30from docmodel import * 31 32exclude_funcs = ['krb5_free_octet_data'] 33 34class DocNode(object): 35 """ 36 Represents the structure of xml node. 37 """ 38 def __init__(self, name): 39 """ 40 @param node: name - the name of a node. 41 @param attributes: a dictionary populated with attributes of a node 42 @param children: a dictionary with lists of children nodes. Nodes 43 in lists are ordered as they appear in a document. 44 @param content: a content of xml node represented as a list of 45 tuples [(type,value)] with type = ['char'|'element']. 46 If type is 'char' then the value is a character string otherwise 47 it is a reference to a child node. 48 """ 49 self.name = name 50 self.content = list() 51 self.attributes = dict() 52 self.children = defaultdict(list) 53 54 def walk(self, decorators, sub_ws, stack=[]): 55 result = list() 56 decorator = decorators.get(self.name, decorators['default']) 57 stack.append(decorators['default']) 58 decorators['default'] = decorator 59 60 for (obj_type,obj) in self.content: 61 if obj_type == 'char': 62 if obj != '': 63 result.append(obj) 64 else: 65 partial = obj.walk(decorators,1, stack) 66 if partial is not None: 67 result.append(' %s ' % partial) 68 decorators['default'] = stack.pop() 69 result = decorator(self, ''.join(result)) 70 if result is not None: 71 if sub_ws == 1: 72 result = re.sub(r'[ ]+', r' ', result) 73 else: 74 result = result.strip() 75 76 return result 77 78 def getContent(self): 79 decorators = {'default': lambda node,value: value} 80 result = self.walk(decorators, 1) 81 if len(result) == 0: 82 result = None 83 84 return result 85 86 def __repr__(self): 87 result = ['Content: %s' % self.content] 88 89 for (key,value) in self.attributes.iteritems(): 90 result.append('Attr: %s = %s' % (key,value)) 91 for (key,value) in self.children.iteritems(): 92 result.append('Child: %s,%i' % (key,len(value))) 93 94 return '\n'.join(result) 95 96class DoxyContenHandler(ContentHandler): 97 def __init__(self, builder): 98 self.builder = builder 99 self.counters = defaultdict(int) 100 self._nodes = None 101 self._current = None 102 103 def startDocument(self): 104 pass 105 106 def endDocument(self): 107 import sys 108 109 def startElement(self, name, attrs): 110 if name == self.builder.toplevel: 111 self._nodes = [] 112 113 if name == 'memberdef': 114 kind = attrs.get('kind') 115 if kind is None: 116 raise ValueError('Kind is not defined') 117 self.counters[kind] += 1 118 119 if self._nodes is None: 120 return 121 122 node = DocNode(name) 123 for (key,value) in attrs.items(): 124 node.attributes[key] = value 125 if self._current is not None: 126 self._current.children[name].append(node) 127 self._nodes.append(self._current) 128 self._current = node 129 130 def characters(self, content): 131 132 if self._current is not None: 133 self._current.content.append(('char',content.strip())) 134 135 def endElement(self, name): 136 if name == self.builder.toplevel: 137 assert(len(self._nodes) == 0) 138 self._nodes = None 139 self.builder.document.append(self._current) 140 self._current = None 141 else: 142 if self._nodes is not None: 143 node = self._current 144 self._current = self._nodes.pop() 145 self._current.content.append(('element',node)) 146 147 148class XML2AST(object): 149 """ 150 Translates XML document into Abstract Syntax Tree like representation 151 The content of document is stored in self.document 152 """ 153 def __init__(self, xmlpath, toplevel='doxygen'): 154 self.document = list() 155 self.toplevel = toplevel 156 self.parser = make_parser() 157 handler = DoxyContenHandler(self) 158 self.parser.setContentHandler(handler) 159 filename = 'krb5_8hin.xml' 160 filepath = '%s/%s' % (xmlpath,filename) 161 self.parser.parse(open(filepath,'r')) 162 163 164class DoxyFuncs(XML2AST): 165 def __init__(self, path): 166 super(DoxyFuncs, self).__init__(path,toplevel='memberdef') 167 self.objects = list() 168 169 def run(self): 170 for node in self.document: 171 self.process(node) 172 173 def process(self, node): 174 node_type = node.attributes['kind'] 175 if node_type == 'function': 176 data = self._process_function_node(node) 177 else: 178 return 179 180 if 'name' in data and data['name'] in exclude_funcs: 181 return 182 self.objects.append(DocModel(**data)) 183 184 def save(self, templates, target_dir): 185 for obj in self.objects: 186 template_path = templates[obj.category] 187 outpath = '%s/%s.rst' % (target_dir,obj.name) 188 obj.save(outpath, template_path) 189 190 191 def _process_function_node(self, node): 192 f_name = node.children['name'][0].getContent() 193 f_Id = node.attributes['id'] 194 f_ret_type = self._process_type_node(node.children['type'][0]) 195 f_brief = node.children['briefdescription'][0].getContent() 196 f_detailed = node.children['detaileddescription'][0] 197 detailed_description = self._process_description_node(f_detailed) 198 return_value_description = self._process_return_value_description(f_detailed) 199 retval_description = self._process_retval_description(f_detailed) 200 warning_description = self._process_warning_description(f_detailed) 201 seealso_description = self._process_seealso_description(f_detailed) 202 notes_description = self._process_notes_description(f_detailed) 203 f_version = self._process_version_description(f_detailed) 204 deprecated_description = self._process_deprecated_description(f_detailed) 205 param_description_map = self.process_parameter_description(f_detailed) 206 f_definition = node.children['definition'][0].getContent() 207 f_argsstring = node.children['argsstring'][0].getContent() 208 209 function_descr = {'category': 'function', 210 'name': f_name, 211 'Id': f_Id, 212 'return_type': f_ret_type[1], 213 'return_description': return_value_description, 214 'retval_description': retval_description, 215 'sa_description': seealso_description, 216 'warn_description': warning_description, 217 'notes_description': notes_description, 218 'short_description': f_brief, 219 'version_num': f_version, 220 'long_description': detailed_description, 221 'deprecated_description': deprecated_description, 222 'parameters': list()} 223 224 parameters = function_descr['parameters'] 225 for (i,p) in enumerate(node.children['param']): 226 type_node = p.children['type'][0] 227 p_type = self._process_type_node(type_node) 228 if p_type[1].find('...') > -1 : 229 p_name = '' 230 else: 231 p_name = None 232 p_name_node = p.children.get('declname') 233 if p_name_node is not None: 234 p_name = p_name_node[0].getContent() 235 (p_direction,p_descr) = param_description_map.get(p_name,(None,None)) 236 237 param_descr = {'seqno': i, 238 'name': p_name, 239 'direction': p_direction, 240 'type': p_type[1], 241 'typeId': p_type[0], 242 'description': p_descr} 243 parameters.append(param_descr) 244 result = Function(**function_descr) 245 print(result, file=self.tmp) 246 247 return function_descr 248 249 def _process_type_node(self, type_node): 250 """ 251 Type node has form 252 <type>type_string</type> 253 for build in types and 254 <type> 255 <ref refid='reference',kindref='member|compound'> 256 'type_name' 257 </ref></type> 258 postfix (ex. *, **m, etc.) 259 </type> 260 for user defined types. 261 """ 262 type_ref_node = type_node.children.get('ref') 263 if type_ref_node is not None: 264 p_type_id = type_ref_node[0].attributes['refid'] 265 else: 266 p_type_id = None 267 p_type = type_node.getContent() 268 # remove some macros 269 p_type = re.sub('KRB5_ATTR_DEPRECATED', '', p_type) 270 p_type = re.sub('KRB5_CALLCONV_C', '', p_type) 271 p_type = re.sub('KRB5_CALLCONV_WRONG', '', p_type) 272 p_type = re.sub('KRB5_CALLCONV', '', p_type) 273 p_type = p_type.strip() 274 275 return (p_type_id, p_type) 276 277 def _process_description_node(self, node): 278 """ 279 Description node is comprised of <para>...</para> sections 280 """ 281 para = node.children.get('para') 282 result = list() 283 if para is not None: 284 decorators = {'default': self.paragraph_content_decorator} 285 for e in para: 286 result.append(str(e.walk(decorators, 1))) 287 result.append('\n') 288 result = '\n'.join(result) 289 290 return result 291 292 def return_value_description_decorator(self, node, value): 293 if node.name == 'simplesect': 294 if node.attributes['kind'] == 'return': 295 cont = set() 296 cont = node.getContent() 297 return value 298 else: 299 return None 300 301 def paragraph_content_decorator(self, node, value): 302 if node.name == 'para': 303 return value + '\n' 304 elif node.name == 'simplesect': 305 if node.attributes['kind'] == 'return': 306 return None 307 elif node.name == 'ref': 308 if value.find('()') >= 0: 309 # functions 310 return ':c:func:' + '`' + value + '`' 311 else: 312 # macro's 313 return ':data:' + '`' + value + '`' 314 elif node.name == 'emphasis': 315 return '*' + value + '*' 316 elif node.name == 'itemizedlist': 317 return '\n' + value 318 elif node.name == 'listitem': 319 return '\n\t - ' + value + '\n' 320 elif node.name == 'computeroutput': 321 return '**' + value + '**' 322 else: 323 return None 324 325 def parameter_name_decorator(self, node, value): 326 if node.name == 'parametername': 327 direction = node.attributes.get('direction') 328 if direction is not None: 329 value = '%s:%s' % (value,direction) 330 return value 331 332 elif node.name == 'parameterdescription': 333 return None 334 else: 335 return value 336 337 def parameter_description_decorator(self, node, value): 338 if node.name == 'parameterdescription': 339 return value 340 elif node.name == 'parametername': 341 return None 342 else: 343 return value 344 345 def process_parameter_description(self, node): 346 """ 347 Parameter descriptions reside inside detailed description section. 348 """ 349 para = node.children.get('para') 350 result = dict() 351 if para is not None: 352 for e in para: 353 354 param_list = e.children.get('parameterlist') 355 if param_list is None: 356 continue 357 param_items = param_list[0].children.get('parameteritem') 358 if param_items is None: 359 continue 360 for it in param_items: 361 decorators = {'default': self.parameter_name_decorator} 362 direction = None 363 name = it.walk(decorators,0).split(':') 364 if len(name) == 2: 365 direction = name[1] 366 367 decorators = {'default': self.parameter_description_decorator, 368 'para': self.paragraph_content_decorator} 369 description = it.walk(decorators, 0) 370 result[name[0]] = (direction,description) 371 return result 372 373 374 def _process_return_value_description(self, node): 375 result = None 376 ret = list() 377 378 para = node.children.get('para') 379 if para is not None: 380 for p in para: 381 simplesect_list = p.children.get('simplesect') 382 if simplesect_list is None: 383 continue 384 for it in simplesect_list: 385 decorators = {'default': self.return_value_description_decorator, 386 'para': self.parameter_name_decorator} 387 result = it.walk(decorators, 1) 388 if result is not None: 389 ret.append(result) 390 return ret 391 392 393 def _process_retval_description(self, node): 394 """ 395 retval descriptions reside inside detailed description section. 396 """ 397 para = node.children.get('para') 398 399 result = None 400 ret = list() 401 if para is not None: 402 403 for e in para: 404 param_list = e.children.get('parameterlist') 405 if param_list is None: 406 continue 407 for p in param_list: 408 kind = p.attributes['kind'] 409 if kind == 'retval': 410 411 param_items = p.children.get('parameteritem') 412 if param_items is None: 413 continue 414 415 416 for it in param_items: 417 param_descr = it.children.get('parameterdescription') 418 if param_descr is not None: 419 val = param_descr[0].children.get('para') 420 421 if val is not None: 422 val_descr = val[0].getContent() 423 424 else: 425 val_descr ='' 426 427 decorators = {'default': self.parameter_name_decorator} 428 429 name = it.walk(decorators, 1).split(':') 430 431 val = name[0] 432 result = " %s %s" % (val, val_descr) 433 ret.append (result) 434 return ret 435 436 def return_warning_decorator(self, node, value): 437 if node.name == 'simplesect': 438 if node.attributes['kind'] == 'warning': 439 return value 440 else: 441 return None 442 443 def _process_warning_description(self, node): 444 result = None 445 para = node.children.get('para') 446 if para is not None: 447 for p in para: 448 simplesect_list = p.children.get('simplesect') 449 if simplesect_list is None: 450 continue 451 for it in simplesect_list: 452 decorators = {'default': self.return_warning_decorator, 453 'para': self.paragraph_content_decorator} 454 result = it.walk(decorators, 1) 455 # Assuming that only one Warning per function 456 if result is not None: 457 return result 458 return result 459 460 def return_seealso_decorator(self, node, value): 461 if node.name == 'simplesect': 462 if node.attributes['kind'] == 'see': 463 return value 464 else: 465 return None 466 467 def _process_seealso_description(self, node): 468 result = None 469 para = node.children.get('para') 470 if para is not None: 471 for p in para: 472 simplesect_list = p.children.get('simplesect') 473 if simplesect_list is None: 474 continue 475 for it in simplesect_list: 476 decorators = {'default': self.return_seealso_decorator, 477 'para': self.paragraph_content_decorator} 478 result = it.walk(decorators, 1) 479 return result 480 481 def return_version_decorator(self, node, value): 482 if node.name == 'simplesect': 483 if node.attributes['kind'] == 'version': 484 return value 485 else: 486 return None 487 488 def _process_version_description(self, node): 489 result = None 490 para = node.children.get('para') 491 if para is not None: 492 for p in para: 493 simplesect_list = p.children.get('simplesect') 494 if simplesect_list is None: 495 continue 496 for it in simplesect_list: 497 decorators = {'default': self.return_version_decorator, 498 'para': self.paragraph_content_decorator} 499 result = it.walk(decorators, 1) 500 if result is not None: 501 return result 502 return result 503 504 def return_notes_decorator(self, node, value): 505 if node.name == 'simplesect': 506 if node.attributes['kind'] == 'note': 507 # We indent notes with an extra tab. Do it for all paragraphs. 508 return value.replace("\n ", "\n\n\t "); 509 else: 510 return None 511 512 def _process_notes_description(self, node): 513 result = None 514 para = node.children.get('para') 515 if para is not None: 516 for p in para: 517 simplesect_list = p.children.get('simplesect') 518 if simplesect_list is None: 519 continue 520 for it in simplesect_list: 521 decorators = {'default': self.return_notes_decorator, 522 'para': self.paragraph_content_decorator} 523 result = it.walk(decorators, 1) 524 if result is not None: 525 return result 526 return result 527 528 def return_deprecated_decorator(self, node, value): 529 if node.name == 'xrefsect': 530 if node.attributes['id'].find('deprecated_') > -1: 531 xreftitle = node.children.get('xreftitle') 532 if xreftitle[0] is not None: 533 xrefdescr = node.children.get('xrefdescription') 534 deprecated_descr = "DEPRECATED %s" % xrefdescr[0].getContent() 535 return deprecated_descr 536 else: 537 return None 538 539 def _process_deprecated_description(self, node): 540 result = None 541 para = node.children.get('para') 542 if para is not None: 543 for p in para: 544 xrefsect_list = p.children.get('xrefsect') 545 if xrefsect_list is None: 546 continue 547 for it in xrefsect_list: 548 decorators = {'default': self.return_deprecated_decorator, 549 'para': self.paragraph_content_decorator} 550 result = it.walk(decorators, 1) 551 if result is not None: 552 return result 553 return result 554 555 def break_into_lines(self, value, linelen=82): 556 breaks = range(0,len(value),linelen) + [len(value)] 557 result = list() 558 for (start,end) in zip(breaks[:-1],breaks[1:]): 559 result.append(value[start:end]) 560 result = '\n'.join(result) 561 562 return result 563 564 def _save(self, table, path = None): 565 if path is None: 566 f = sys.stdout 567 else: 568 f = open(path, 'w') 569 for l in table: 570 f.write('%s\n' % ','.join(l)) 571 if path is not None: 572 f.close() 573 574 575 576class DoxyBuilderFuncs(DoxyFuncs): 577 def __init__(self, xmlpath, rstpath): 578 super(DoxyBuilderFuncs,self).__init__(xmlpath) 579 self.target_dir = rstpath 580 outfile = '%s/%s' % (self.target_dir, 'out.txt') 581 self.tmp = open(outfile, 'w') 582 583 def run_all(self): 584 self.run() 585 templates = {'function': 'func_document.tmpl'} 586 self.save(templates, self.target_dir) 587 588 def test_run(self): 589 self.run() 590 591if __name__ == '__main__': 592 builder = DoxyBuilderFuncs(xmlpath, rstpath) 593 builder.run_all() 594 595