1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Script to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This script performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in three big parts: 16 1) RST formatters: Use to convert a string to a RST output 17 2) Parser helpers: Functions to parse the YAML data structure 18 3) Main function and small helpers 19""" 20 21from typing import Any, Dict, List 22import os.path 23import sys 24import argparse 25import logging 26import yaml 27 28 29SPACE_PER_LEVEL = 4 30 31 32# RST Formatters 33# ============== 34def headroom(level: int) -> str: 35 """Return space to format""" 36 return " " * (level * SPACE_PER_LEVEL) 37 38 39def bold(text: str) -> str: 40 """Format bold text""" 41 return f"**{text}**" 42 43 44def inline(text: str) -> str: 45 """Format inline text""" 46 return f"``{text}``" 47 48 49def sanitize(text: str) -> str: 50 """Remove newlines and multiple spaces""" 51 # This is useful for some fields that are spread across multiple lines 52 return str(text).replace("\n", " ").strip() 53 54 55def rst_fields(key: str, value: str, level: int = 0) -> str: 56 """Return a RST formatted field""" 57 return headroom(level) + f":{key}: {value}" 58 59 60def rst_definition(key: str, value: Any, level: int = 0) -> str: 61 """Format a single rst definition""" 62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value) 63 64 65def rst_paragraph(paragraph: str, level: int = 0) -> str: 66 """Return a formatted paragraph""" 67 return headroom(level) + paragraph 68 69 70def rst_bullet(item: str, level: int = 0) -> str: 71 """Return a formatted a bullet""" 72 return headroom(level) + f"- {item}" 73 74 75def rst_subsection(title: str) -> str: 76 """Add a sub-section to the document""" 77 return f"{title}\n" + "-" * len(title) 78 79 80def rst_subsubsection(title: str) -> str: 81 """Add a sub-sub-section to the document""" 82 return f"{title}\n" + "~" * len(title) 83 84 85def rst_section(namespace: str, prefix: str, title: str) -> str: 86 """Add a section to the document""" 87 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 88 89 90def rst_subtitle(title: str) -> str: 91 """Add a subtitle to the document""" 92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 93 94 95def rst_title(title: str) -> str: 96 """Add a title to the document""" 97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 98 99 100def rst_list_inline(list_: List[str], level: int = 0) -> str: 101 """Format a list using inlines""" 102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]" 103 104 105def rst_ref(namespace: str, prefix: str, name: str) -> str: 106 """Add a hyperlink to the document""" 107 mappings = {'enum': 'definition', 108 'fixed-header': 'definition', 109 'nested-attributes': 'attribute-set', 110 'struct': 'definition'} 111 if prefix in mappings: 112 prefix = mappings[prefix] 113 return f":ref:`{namespace}-{prefix}-{name}`" 114 115 116def rst_header() -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 126def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 136def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 141# Parsers 142# ======= 143 144 145def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str: 146 """Parse 'multicast' group list and return a formatted string""" 147 lines = [] 148 for group in mcast_group: 149 lines.append(rst_bullet(group["name"])) 150 151 return "\n".join(lines) 152 153 154def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str: 155 """Parse 'do' section and return a formatted string""" 156 lines = [] 157 for key in do_dict.keys(): 158 lines.append(rst_paragraph(bold(key), level + 1)) 159 if key in ['request', 'reply']: 160 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n") 161 else: 162 lines.append(headroom(level + 2) + do_dict[key] + "\n") 163 164 return "\n".join(lines) 165 166 167def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str: 168 """Parse 'attributes' section""" 169 if "attributes" not in attrs: 170 return "" 171 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)] 172 173 return "\n".join(lines) 174 175 176def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str: 177 """Parse operations block""" 178 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 179 linkable = ["fixed-header", "attribute-set"] 180 lines = [] 181 182 for operation in operations: 183 lines.append(rst_section(namespace, 'operation', operation["name"])) 184 lines.append(rst_paragraph(operation["doc"]) + "\n") 185 186 for key in operation.keys(): 187 if key in preprocessed: 188 # Skip the special fields 189 continue 190 value = operation[key] 191 if key in linkable: 192 value = rst_ref(namespace, key, value) 193 lines.append(rst_fields(key, value, 0)) 194 if 'flags' in operation: 195 lines.append(rst_fields('flags', rst_list_inline(operation['flags']))) 196 197 if "do" in operation: 198 lines.append(rst_paragraph(":do:", 0)) 199 lines.append(parse_do(operation["do"], 0)) 200 if "dump" in operation: 201 lines.append(rst_paragraph(":dump:", 0)) 202 lines.append(parse_do(operation["dump"], 0)) 203 204 # New line after fields 205 lines.append("\n") 206 207 return "\n".join(lines) 208 209 210def parse_entries(entries: List[Dict[str, Any]], level: int) -> str: 211 """Parse a list of entries""" 212 ignored = ["pad"] 213 lines = [] 214 for entry in entries: 215 if isinstance(entry, dict): 216 # entries could be a list or a dictionary 217 field_name = entry.get("name", "") 218 if field_name in ignored: 219 continue 220 type_ = entry.get("type") 221 if type_: 222 field_name += f" ({inline(type_)})" 223 lines.append( 224 rst_fields(field_name, sanitize(entry.get("doc", "")), level) 225 ) 226 elif isinstance(entry, list): 227 lines.append(rst_list_inline(entry, level)) 228 else: 229 lines.append(rst_bullet(inline(sanitize(entry)), level)) 230 231 lines.append("\n") 232 return "\n".join(lines) 233 234 235def parse_definitions(defs: Dict[str, Any], namespace: str) -> str: 236 """Parse definitions section""" 237 preprocessed = ["name", "entries", "members"] 238 ignored = ["render-max"] # This is not printed 239 lines = [] 240 241 for definition in defs: 242 lines.append(rst_section(namespace, 'definition', definition["name"])) 243 for k in definition.keys(): 244 if k in preprocessed + ignored: 245 continue 246 lines.append(rst_fields(k, sanitize(definition[k]), 0)) 247 248 # Field list needs to finish with a new line 249 lines.append("\n") 250 if "entries" in definition: 251 lines.append(rst_paragraph(":entries:", 0)) 252 lines.append(parse_entries(definition["entries"], 1)) 253 if "members" in definition: 254 lines.append(rst_paragraph(":members:", 0)) 255 lines.append(parse_entries(definition["members"], 1)) 256 257 return "\n".join(lines) 258 259 260def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str: 261 """Parse attribute from attribute-set""" 262 preprocessed = ["name", "type"] 263 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 264 ignored = ["checks"] 265 lines = [] 266 267 for entry in entries: 268 lines.append(rst_section(namespace, 'attribute-set', entry["name"])) 269 for attr in entry["attributes"]: 270 type_ = attr.get("type") 271 attr_line = attr["name"] 272 if type_: 273 # Add the attribute type in the same line 274 attr_line += f" ({inline(type_)})" 275 276 lines.append(rst_subsubsection(attr_line)) 277 278 for k in attr.keys(): 279 if k in preprocessed + ignored: 280 continue 281 if k in linkable: 282 value = rst_ref(namespace, k, attr[k]) 283 else: 284 value = sanitize(attr[k]) 285 lines.append(rst_fields(k, value, 0)) 286 lines.append("\n") 287 288 return "\n".join(lines) 289 290 291def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str: 292 """Parse sub-message definitions""" 293 lines = [] 294 295 for entry in entries: 296 lines.append(rst_section(namespace, 'sub-message', entry["name"])) 297 for fmt in entry["formats"]: 298 value = fmt["value"] 299 300 lines.append(rst_bullet(bold(value))) 301 for attr in ['fixed-header', 'attribute-set']: 302 if attr in fmt: 303 lines.append(rst_fields(attr, 304 rst_ref(namespace, attr, fmt[attr]), 305 1)) 306 lines.append("\n") 307 308 return "\n".join(lines) 309 310 311def parse_yaml(obj: Dict[str, Any]) -> str: 312 """Format the whole YAML into a RST string""" 313 lines = [] 314 315 # Main header 316 317 family = obj['name'] 318 319 lines.append(rst_header()) 320 lines.append(rst_label("netlink-" + family)) 321 322 title = f"Family ``{family}`` netlink specification" 323 lines.append(rst_title(title)) 324 lines.append(rst_paragraph(".. contents:: :depth: 3\n")) 325 326 if "doc" in obj: 327 lines.append(rst_subtitle("Summary")) 328 lines.append(rst_paragraph(obj["doc"], 0)) 329 330 # Operations 331 if "operations" in obj: 332 lines.append(rst_subtitle("Operations")) 333 lines.append(parse_operations(obj["operations"]["list"], family)) 334 335 # Multicast groups 336 if "mcast-groups" in obj: 337 lines.append(rst_subtitle("Multicast groups")) 338 lines.append(parse_mcast_group(obj["mcast-groups"]["list"])) 339 340 # Definitions 341 if "definitions" in obj: 342 lines.append(rst_subtitle("Definitions")) 343 lines.append(parse_definitions(obj["definitions"], family)) 344 345 # Attributes set 346 if "attribute-sets" in obj: 347 lines.append(rst_subtitle("Attribute sets")) 348 lines.append(parse_attr_sets(obj["attribute-sets"], family)) 349 350 # Sub-messages 351 if "sub-messages" in obj: 352 lines.append(rst_subtitle("Sub-messages")) 353 lines.append(parse_sub_messages(obj["sub-messages"], family)) 354 355 return "\n".join(lines) 356 357 358# Main functions 359# ============== 360 361 362def parse_arguments() -> argparse.Namespace: 363 """Parse arguments from user""" 364 parser = argparse.ArgumentParser(description="Netlink RST generator") 365 366 parser.add_argument("-v", "--verbose", action="store_true") 367 parser.add_argument("-o", "--output", help="Output file name") 368 369 # Index and input are mutually exclusive 370 group = parser.add_mutually_exclusive_group() 371 group.add_argument( 372 "-x", "--index", action="store_true", help="Generate the index page" 373 ) 374 group.add_argument("-i", "--input", help="YAML file name") 375 376 args = parser.parse_args() 377 378 if args.verbose: 379 logging.basicConfig(level=logging.DEBUG) 380 381 if args.input and not os.path.isfile(args.input): 382 logging.warning("%s is not a valid file.", args.input) 383 sys.exit(-1) 384 385 if not args.output: 386 logging.error("No output file specified.") 387 sys.exit(-1) 388 389 if os.path.isfile(args.output): 390 logging.debug("%s already exists. Overwriting it.", args.output) 391 392 return args 393 394 395def parse_yaml_file(filename: str) -> str: 396 """Transform the YAML specified by filename into an RST-formatted string""" 397 with open(filename, "r", encoding="utf-8") as spec_file: 398 yaml_data = yaml.safe_load(spec_file) 399 content = parse_yaml(yaml_data) 400 401 return content 402 403 404def write_to_rstfile(content: str, filename: str) -> None: 405 """Write the generated content into an RST file""" 406 logging.debug("Saving RST file to %s", filename) 407 408 with open(filename, "w", encoding="utf-8") as rst_file: 409 rst_file.write(content) 410 411 412def generate_main_index_rst(output: str) -> None: 413 """Generate the `networking_spec/index` content and write to the file""" 414 lines = [] 415 416 lines.append(rst_header()) 417 lines.append(rst_label("specs")) 418 lines.append(rst_title("Netlink Family Specifications")) 419 lines.append(rst_toctree(1)) 420 421 index_dir = os.path.dirname(output) 422 logging.debug("Looking for .rst files in %s", index_dir) 423 for filename in sorted(os.listdir(index_dir)): 424 if not filename.endswith(".rst") or filename == "index.rst": 425 continue 426 lines.append(f" {filename.replace('.rst', '')}\n") 427 428 logging.debug("Writing an index file at %s", output) 429 write_to_rstfile("".join(lines), output) 430 431 432def main() -> None: 433 """Main function that reads the YAML files and generates the RST files""" 434 435 args = parse_arguments() 436 437 if args.input: 438 logging.debug("Parsing %s", args.input) 439 try: 440 content = parse_yaml_file(os.path.join(args.input)) 441 except Exception as exception: 442 logging.warning("Failed to parse %s.", args.input) 443 logging.warning(exception) 444 sys.exit(-1) 445 446 write_to_rstfile(content, args.output) 447 448 if args.index: 449 # Generate the index RST file 450 generate_main_index_rst(args.output) 451 452 453if __name__ == "__main__": 454 main() 455