1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Script to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This script performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in three big parts: 16 1) RST formatters: Use to convert a string to a RST output 17 2) Parser helpers: Functions to parse the YAML data structure 18 3) Main function and small helpers 19""" 20 21from typing import Any, Dict, List 22import os.path 23import sys 24import argparse 25import logging 26import yaml 27 28 29SPACE_PER_LEVEL = 4 30 31 32# RST Formatters 33# ============== 34def headroom(level: int) -> str: 35 """Return space to format""" 36 return " " * (level * SPACE_PER_LEVEL) 37 38 39def bold(text: str) -> str: 40 """Format bold text""" 41 return f"**{text}**" 42 43 44def inline(text: str) -> str: 45 """Format inline text""" 46 return f"``{text}``" 47 48 49def sanitize(text: str) -> str: 50 """Remove newlines and multiple spaces""" 51 # This is useful for some fields that are spread across multiple lines 52 return str(text).replace("\n", " ").strip() 53 54 55def rst_fields(key: str, value: str, level: int = 0) -> str: 56 """Return a RST formatted field""" 57 return headroom(level) + f":{key}: {value}" 58 59 60def rst_definition(key: str, value: Any, level: int = 0) -> str: 61 """Format a single rst definition""" 62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value) 63 64 65def rst_paragraph(paragraph: str, level: int = 0) -> str: 66 """Return a formatted paragraph""" 67 return headroom(level) + paragraph 68 69 70def rst_bullet(item: str, level: int = 0) -> str: 71 """Return a formatted a bullet""" 72 return headroom(level) + f"- {item}" 73 74 75def rst_subsection(title: str) -> str: 76 """Add a sub-section to the document""" 77 return f"{title}\n" + "-" * len(title) 78 79 80def rst_subsubsection(title: str) -> str: 81 """Add a sub-sub-section to the document""" 82 return f"{title}\n" + "~" * len(title) 83 84 85def rst_section(namespace: str, prefix: str, title: str) -> str: 86 """Add a section to the document""" 87 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 88 89 90def rst_subtitle(title: str) -> str: 91 """Add a subtitle to the document""" 92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 93 94 95def rst_title(title: str) -> str: 96 """Add a title to the document""" 97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 98 99 100def rst_list_inline(list_: List[str], level: int = 0) -> str: 101 """Format a list using inlines""" 102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]" 103 104 105def rst_ref(namespace: str, prefix: str, name: str) -> str: 106 """Add a hyperlink to the document""" 107 mappings = {'enum': 'definition', 108 'fixed-header': 'definition', 109 'nested-attributes': 'attribute-set', 110 'struct': 'definition'} 111 if prefix in mappings: 112 prefix = mappings[prefix] 113 return f":ref:`{namespace}-{prefix}-{name}`" 114 115 116def rst_header() -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 126def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 136def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 141# Parsers 142# ======= 143 144 145def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str: 146 """Parse 'multicast' group list and return a formatted string""" 147 lines = [] 148 for group in mcast_group: 149 lines.append(rst_bullet(group["name"])) 150 151 return "\n".join(lines) 152 153 154def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str: 155 """Parse 'do' section and return a formatted string""" 156 lines = [] 157 for key in do_dict.keys(): 158 lines.append(rst_paragraph(bold(key), level + 1)) 159 if key in ['request', 'reply']: 160 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n") 161 else: 162 lines.append(headroom(level + 2) + do_dict[key] + "\n") 163 164 return "\n".join(lines) 165 166 167def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str: 168 """Parse 'attributes' section""" 169 if "attributes" not in attrs: 170 return "" 171 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)] 172 173 return "\n".join(lines) 174 175 176def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str: 177 """Parse operations block""" 178 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 179 linkable = ["fixed-header", "attribute-set"] 180 lines = [] 181 182 for operation in operations: 183 lines.append(rst_section(namespace, 'operation', operation["name"])) 184 lines.append(rst_paragraph(operation["doc"]) + "\n") 185 186 for key in operation.keys(): 187 if key in preprocessed: 188 # Skip the special fields 189 continue 190 value = operation[key] 191 if key in linkable: 192 value = rst_ref(namespace, key, value) 193 lines.append(rst_fields(key, value, 0)) 194 if 'flags' in operation: 195 lines.append(rst_fields('flags', rst_list_inline(operation['flags']))) 196 197 if "do" in operation: 198 lines.append(rst_paragraph(":do:", 0)) 199 lines.append(parse_do(operation["do"], 0)) 200 if "dump" in operation: 201 lines.append(rst_paragraph(":dump:", 0)) 202 lines.append(parse_do(operation["dump"], 0)) 203 204 # New line after fields 205 lines.append("\n") 206 207 return "\n".join(lines) 208 209 210def parse_entries(entries: List[Dict[str, Any]], level: int) -> str: 211 """Parse a list of entries""" 212 ignored = ["pad"] 213 lines = [] 214 for entry in entries: 215 if isinstance(entry, dict): 216 # entries could be a list or a dictionary 217 field_name = entry.get("name", "") 218 if field_name in ignored: 219 continue 220 type_ = entry.get("type") 221 if type_: 222 field_name += f" ({inline(type_)})" 223 lines.append( 224 rst_fields(field_name, sanitize(entry.get("doc", "")), level) 225 ) 226 elif isinstance(entry, list): 227 lines.append(rst_list_inline(entry, level)) 228 else: 229 lines.append(rst_bullet(inline(sanitize(entry)), level)) 230 231 lines.append("\n") 232 return "\n".join(lines) 233 234 235def parse_definitions(defs: Dict[str, Any], namespace: str) -> str: 236 """Parse definitions section""" 237 preprocessed = ["name", "entries", "members"] 238 ignored = ["render-max"] # This is not printed 239 lines = [] 240 241 for definition in defs: 242 lines.append(rst_section(namespace, 'definition', definition["name"])) 243 for k in definition.keys(): 244 if k in preprocessed + ignored: 245 continue 246 lines.append(rst_fields(k, sanitize(definition[k]), 0)) 247 248 # Field list needs to finish with a new line 249 lines.append("\n") 250 if "entries" in definition: 251 lines.append(rst_paragraph(":entries:", 0)) 252 lines.append(parse_entries(definition["entries"], 1)) 253 if "members" in definition: 254 lines.append(rst_paragraph(":members:", 0)) 255 lines.append(parse_entries(definition["members"], 1)) 256 257 return "\n".join(lines) 258 259 260def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str: 261 """Parse attribute from attribute-set""" 262 preprocessed = ["name", "type"] 263 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 264 ignored = ["checks"] 265 lines = [] 266 267 for entry in entries: 268 lines.append(rst_section(namespace, 'attribute-set', entry["name"])) 269 for attr in entry["attributes"]: 270 type_ = attr.get("type") 271 attr_line = attr["name"] 272 if type_: 273 # Add the attribute type in the same line 274 attr_line += f" ({inline(type_)})" 275 276 lines.append(rst_subsubsection(attr_line)) 277 278 for k in attr.keys(): 279 if k in preprocessed + ignored: 280 continue 281 if k in linkable: 282 value = rst_ref(namespace, k, attr[k]) 283 else: 284 value = sanitize(attr[k]) 285 lines.append(rst_fields(k, value, 0)) 286 lines.append("\n") 287 288 return "\n".join(lines) 289 290 291def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str: 292 """Parse sub-message definitions""" 293 lines = [] 294 295 for entry in entries: 296 lines.append(rst_section(namespace, 'sub-message', entry["name"])) 297 for fmt in entry["formats"]: 298 value = fmt["value"] 299 300 lines.append(rst_bullet(bold(value))) 301 for attr in ['fixed-header', 'attribute-set']: 302 if attr in fmt: 303 lines.append(rst_fields(attr, 304 rst_ref(namespace, attr, fmt[attr]), 305 1)) 306 lines.append("\n") 307 308 return "\n".join(lines) 309 310 311def parse_yaml(obj: Dict[str, Any]) -> str: 312 """Format the whole YAML into a RST string""" 313 lines = [] 314 315 # Main header 316 317 lines.append(rst_header()) 318 319 family = obj['name'] 320 321 title = f"Family ``{family}`` netlink specification" 322 lines.append(rst_title(title)) 323 lines.append(rst_paragraph(".. contents:: :depth: 3\n")) 324 325 if "doc" in obj: 326 lines.append(rst_subtitle("Summary")) 327 lines.append(rst_paragraph(obj["doc"], 0)) 328 329 # Operations 330 if "operations" in obj: 331 lines.append(rst_subtitle("Operations")) 332 lines.append(parse_operations(obj["operations"]["list"], family)) 333 334 # Multicast groups 335 if "mcast-groups" in obj: 336 lines.append(rst_subtitle("Multicast groups")) 337 lines.append(parse_mcast_group(obj["mcast-groups"]["list"])) 338 339 # Definitions 340 if "definitions" in obj: 341 lines.append(rst_subtitle("Definitions")) 342 lines.append(parse_definitions(obj["definitions"], family)) 343 344 # Attributes set 345 if "attribute-sets" in obj: 346 lines.append(rst_subtitle("Attribute sets")) 347 lines.append(parse_attr_sets(obj["attribute-sets"], family)) 348 349 # Sub-messages 350 if "sub-messages" in obj: 351 lines.append(rst_subtitle("Sub-messages")) 352 lines.append(parse_sub_messages(obj["sub-messages"], family)) 353 354 return "\n".join(lines) 355 356 357# Main functions 358# ============== 359 360 361def parse_arguments() -> argparse.Namespace: 362 """Parse arguments from user""" 363 parser = argparse.ArgumentParser(description="Netlink RST generator") 364 365 parser.add_argument("-v", "--verbose", action="store_true") 366 parser.add_argument("-o", "--output", help="Output file name") 367 368 # Index and input are mutually exclusive 369 group = parser.add_mutually_exclusive_group() 370 group.add_argument( 371 "-x", "--index", action="store_true", help="Generate the index page" 372 ) 373 group.add_argument("-i", "--input", help="YAML file name") 374 375 args = parser.parse_args() 376 377 if args.verbose: 378 logging.basicConfig(level=logging.DEBUG) 379 380 if args.input and not os.path.isfile(args.input): 381 logging.warning("%s is not a valid file.", args.input) 382 sys.exit(-1) 383 384 if not args.output: 385 logging.error("No output file specified.") 386 sys.exit(-1) 387 388 if os.path.isfile(args.output): 389 logging.debug("%s already exists. Overwriting it.", args.output) 390 391 return args 392 393 394def parse_yaml_file(filename: str) -> str: 395 """Transform the YAML specified by filename into a rst-formmated string""" 396 with open(filename, "r", encoding="utf-8") as spec_file: 397 yaml_data = yaml.safe_load(spec_file) 398 content = parse_yaml(yaml_data) 399 400 return content 401 402 403def write_to_rstfile(content: str, filename: str) -> None: 404 """Write the generated content into an RST file""" 405 logging.debug("Saving RST file to %s", filename) 406 407 with open(filename, "w", encoding="utf-8") as rst_file: 408 rst_file.write(content) 409 410 411def generate_main_index_rst(output: str) -> None: 412 """Generate the `networking_spec/index` content and write to the file""" 413 lines = [] 414 415 lines.append(rst_header()) 416 lines.append(rst_label("specs")) 417 lines.append(rst_title("Netlink Family Specifications")) 418 lines.append(rst_toctree(1)) 419 420 index_dir = os.path.dirname(output) 421 logging.debug("Looking for .rst files in %s", index_dir) 422 for filename in sorted(os.listdir(index_dir)): 423 if not filename.endswith(".rst") or filename == "index.rst": 424 continue 425 lines.append(f" {filename.replace('.rst', '')}\n") 426 427 logging.debug("Writing an index file at %s", output) 428 write_to_rstfile("".join(lines), output) 429 430 431def main() -> None: 432 """Main function that reads the YAML files and generates the RST files""" 433 434 args = parse_arguments() 435 436 if args.input: 437 logging.debug("Parsing %s", args.input) 438 try: 439 content = parse_yaml_file(os.path.join(args.input)) 440 except Exception as exception: 441 logging.warning("Failed to parse %s.", args.input) 442 logging.warning(exception) 443 sys.exit(-1) 444 445 write_to_rstfile(content, args.output) 446 447 if args.index: 448 # Generate the index RST file 449 generate_main_index_rst(args.output) 450 451 452if __name__ == "__main__": 453 main() 454