1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Script to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This script performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in three big parts: 16 1) RST formatters: Use to convert a string to a RST output 17 2) Parser helpers: Functions to parse the YAML data structure 18 3) Main function and small helpers 19""" 20 21from typing import Any, Dict, List 22import os.path 23import sys 24import argparse 25import logging 26import yaml 27 28 29SPACE_PER_LEVEL = 4 30 31 32# RST Formatters 33# ============== 34def headroom(level: int) -> str: 35 """Return space to format""" 36 return " " * (level * SPACE_PER_LEVEL) 37 38 39def bold(text: str) -> str: 40 """Format bold text""" 41 return f"**{text}**" 42 43 44def inline(text: str) -> str: 45 """Format inline text""" 46 return f"``{text}``" 47 48 49def sanitize(text: str) -> str: 50 """Remove newlines and multiple spaces""" 51 # This is useful for some fields that are spread across multiple lines 52 return str(text).replace("\n", "").strip() 53 54 55def rst_fields(key: str, value: str, level: int = 0) -> str: 56 """Return a RST formatted field""" 57 return headroom(level) + f":{key}: {value}" 58 59 60def rst_definition(key: str, value: Any, level: int = 0) -> str: 61 """Format a single rst definition""" 62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value) 63 64 65def rst_paragraph(paragraph: str, level: int = 0) -> str: 66 """Return a formatted paragraph""" 67 return headroom(level) + paragraph 68 69 70def rst_bullet(item: str, level: int = 0) -> str: 71 """Return a formatted a bullet""" 72 return headroom(level) + f"- {item}" 73 74 75def rst_subsection(title: str) -> str: 76 """Add a sub-section to the document""" 77 return f"{title}\n" + "-" * len(title) 78 79 80def rst_subsubsection(title: str) -> str: 81 """Add a sub-sub-section to the document""" 82 return f"{title}\n" + "~" * len(title) 83 84 85def rst_section(title: str) -> str: 86 """Add a section to the document""" 87 return f"\n{title}\n" + "=" * len(title) 88 89 90def rst_subtitle(title: str) -> str: 91 """Add a subtitle to the document""" 92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 93 94 95def rst_title(title: str) -> str: 96 """Add a title to the document""" 97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 98 99 100def rst_list_inline(list_: List[str], level: int = 0) -> str: 101 """Format a list using inlines""" 102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]" 103 104 105def rst_header() -> str: 106 """The headers for all the auto generated RST files""" 107 lines = [] 108 109 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 110 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 111 112 return "\n".join(lines) 113 114 115def rst_toctree(maxdepth: int = 2) -> str: 116 """Generate a toctree RST primitive""" 117 lines = [] 118 119 lines.append(".. toctree::") 120 lines.append(f" :maxdepth: {maxdepth}\n\n") 121 122 return "\n".join(lines) 123 124 125def rst_label(title: str) -> str: 126 """Return a formatted label""" 127 return f".. _{title}:\n\n" 128 129 130# Parsers 131# ======= 132 133 134def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str: 135 """Parse 'multicast' group list and return a formatted string""" 136 lines = [] 137 for group in mcast_group: 138 lines.append(rst_bullet(group["name"])) 139 140 return "\n".join(lines) 141 142 143def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str: 144 """Parse 'do' section and return a formatted string""" 145 lines = [] 146 for key in do_dict.keys(): 147 lines.append(rst_paragraph(bold(key), level + 1)) 148 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n") 149 150 return "\n".join(lines) 151 152 153def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str: 154 """Parse 'attributes' section""" 155 if "attributes" not in attrs: 156 return "" 157 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)] 158 159 return "\n".join(lines) 160 161 162def parse_operations(operations: List[Dict[str, Any]]) -> str: 163 """Parse operations block""" 164 preprocessed = ["name", "doc", "title", "do", "dump"] 165 lines = [] 166 167 for operation in operations: 168 lines.append(rst_section(operation["name"])) 169 lines.append(rst_paragraph(sanitize(operation["doc"])) + "\n") 170 171 for key in operation.keys(): 172 if key in preprocessed: 173 # Skip the special fields 174 continue 175 lines.append(rst_fields(key, operation[key], 0)) 176 177 if "do" in operation: 178 lines.append(rst_paragraph(":do:", 0)) 179 lines.append(parse_do(operation["do"], 0)) 180 if "dump" in operation: 181 lines.append(rst_paragraph(":dump:", 0)) 182 lines.append(parse_do(operation["dump"], 0)) 183 184 # New line after fields 185 lines.append("\n") 186 187 return "\n".join(lines) 188 189 190def parse_entries(entries: List[Dict[str, Any]], level: int) -> str: 191 """Parse a list of entries""" 192 ignored = ["pad"] 193 lines = [] 194 for entry in entries: 195 if isinstance(entry, dict): 196 # entries could be a list or a dictionary 197 field_name = entry.get("name", "") 198 if field_name in ignored: 199 continue 200 type_ = entry.get("type") 201 if type_: 202 field_name += f" ({inline(type_)})" 203 lines.append( 204 rst_fields(field_name, sanitize(entry.get("doc", "")), level) 205 ) 206 elif isinstance(entry, list): 207 lines.append(rst_list_inline(entry, level)) 208 else: 209 lines.append(rst_bullet(inline(sanitize(entry)), level)) 210 211 lines.append("\n") 212 return "\n".join(lines) 213 214 215def parse_definitions(defs: Dict[str, Any]) -> str: 216 """Parse definitions section""" 217 preprocessed = ["name", "entries", "members"] 218 ignored = ["render-max"] # This is not printed 219 lines = [] 220 221 for definition in defs: 222 lines.append(rst_section(definition["name"])) 223 for k in definition.keys(): 224 if k in preprocessed + ignored: 225 continue 226 lines.append(rst_fields(k, sanitize(definition[k]), 0)) 227 228 # Field list needs to finish with a new line 229 lines.append("\n") 230 if "entries" in definition: 231 lines.append(rst_paragraph(":entries:", 0)) 232 lines.append(parse_entries(definition["entries"], 1)) 233 if "members" in definition: 234 lines.append(rst_paragraph(":members:", 0)) 235 lines.append(parse_entries(definition["members"], 1)) 236 237 return "\n".join(lines) 238 239 240def parse_attr_sets(entries: List[Dict[str, Any]]) -> str: 241 """Parse attribute from attribute-set""" 242 preprocessed = ["name", "type"] 243 ignored = ["checks"] 244 lines = [] 245 246 for entry in entries: 247 lines.append(rst_section(entry["name"])) 248 for attr in entry["attributes"]: 249 type_ = attr.get("type") 250 attr_line = attr["name"] 251 if type_: 252 # Add the attribute type in the same line 253 attr_line += f" ({inline(type_)})" 254 255 lines.append(rst_subsubsection(attr_line)) 256 257 for k in attr.keys(): 258 if k in preprocessed + ignored: 259 continue 260 lines.append(rst_fields(k, sanitize(attr[k]), 0)) 261 lines.append("\n") 262 263 return "\n".join(lines) 264 265 266def parse_sub_messages(entries: List[Dict[str, Any]]) -> str: 267 """Parse sub-message definitions""" 268 lines = [] 269 270 for entry in entries: 271 lines.append(rst_section(entry["name"])) 272 for fmt in entry["formats"]: 273 value = fmt["value"] 274 275 lines.append(rst_bullet(bold(value))) 276 for attr in ['fixed-header', 'attribute-set']: 277 if attr in fmt: 278 lines.append(rst_fields(attr, fmt[attr], 1)) 279 lines.append("\n") 280 281 return "\n".join(lines) 282 283 284def parse_yaml(obj: Dict[str, Any]) -> str: 285 """Format the whole YAML into a RST string""" 286 lines = [] 287 288 # Main header 289 290 lines.append(rst_header()) 291 292 title = f"Family ``{obj['name']}`` netlink specification" 293 lines.append(rst_title(title)) 294 lines.append(rst_paragraph(".. contents::\n")) 295 296 if "doc" in obj: 297 lines.append(rst_subtitle("Summary")) 298 lines.append(rst_paragraph(obj["doc"], 0)) 299 300 # Operations 301 if "operations" in obj: 302 lines.append(rst_subtitle("Operations")) 303 lines.append(parse_operations(obj["operations"]["list"])) 304 305 # Multicast groups 306 if "mcast-groups" in obj: 307 lines.append(rst_subtitle("Multicast groups")) 308 lines.append(parse_mcast_group(obj["mcast-groups"]["list"])) 309 310 # Definitions 311 if "definitions" in obj: 312 lines.append(rst_subtitle("Definitions")) 313 lines.append(parse_definitions(obj["definitions"])) 314 315 # Attributes set 316 if "attribute-sets" in obj: 317 lines.append(rst_subtitle("Attribute sets")) 318 lines.append(parse_attr_sets(obj["attribute-sets"])) 319 320 # Sub-messages 321 if "sub-messages" in obj: 322 lines.append(rst_subtitle("Sub-messages")) 323 lines.append(parse_sub_messages(obj["sub-messages"])) 324 325 return "\n".join(lines) 326 327 328# Main functions 329# ============== 330 331 332def parse_arguments() -> argparse.Namespace: 333 """Parse arguments from user""" 334 parser = argparse.ArgumentParser(description="Netlink RST generator") 335 336 parser.add_argument("-v", "--verbose", action="store_true") 337 parser.add_argument("-o", "--output", help="Output file name") 338 339 # Index and input are mutually exclusive 340 group = parser.add_mutually_exclusive_group() 341 group.add_argument( 342 "-x", "--index", action="store_true", help="Generate the index page" 343 ) 344 group.add_argument("-i", "--input", help="YAML file name") 345 346 args = parser.parse_args() 347 348 if args.verbose: 349 logging.basicConfig(level=logging.DEBUG) 350 351 if args.input and not os.path.isfile(args.input): 352 logging.warning("%s is not a valid file.", args.input) 353 sys.exit(-1) 354 355 if not args.output: 356 logging.error("No output file specified.") 357 sys.exit(-1) 358 359 if os.path.isfile(args.output): 360 logging.debug("%s already exists. Overwriting it.", args.output) 361 362 return args 363 364 365def parse_yaml_file(filename: str) -> str: 366 """Transform the YAML specified by filename into a rst-formmated string""" 367 with open(filename, "r", encoding="utf-8") as spec_file: 368 yaml_data = yaml.safe_load(spec_file) 369 content = parse_yaml(yaml_data) 370 371 return content 372 373 374def write_to_rstfile(content: str, filename: str) -> None: 375 """Write the generated content into an RST file""" 376 logging.debug("Saving RST file to %s", filename) 377 378 with open(filename, "w", encoding="utf-8") as rst_file: 379 rst_file.write(content) 380 381 382def generate_main_index_rst(output: str) -> None: 383 """Generate the `networking_spec/index` content and write to the file""" 384 lines = [] 385 386 lines.append(rst_header()) 387 lines.append(rst_label("specs")) 388 lines.append(rst_title("Netlink Family Specifications")) 389 lines.append(rst_toctree(1)) 390 391 index_dir = os.path.dirname(output) 392 logging.debug("Looking for .rst files in %s", index_dir) 393 for filename in sorted(os.listdir(index_dir)): 394 if not filename.endswith(".rst") or filename == "index.rst": 395 continue 396 lines.append(f" {filename.replace('.rst', '')}\n") 397 398 logging.debug("Writing an index file at %s", output) 399 write_to_rstfile("".join(lines), output) 400 401 402def main() -> None: 403 """Main function that reads the YAML files and generates the RST files""" 404 405 args = parse_arguments() 406 407 if args.input: 408 logging.debug("Parsing %s", args.input) 409 try: 410 content = parse_yaml_file(os.path.join(args.input)) 411 except Exception as exception: 412 logging.warning("Failed to parse %s.", args.input) 413 logging.warning(exception) 414 sys.exit(-1) 415 416 write_to_rstfile(content, args.output) 417 418 if args.index: 419 # Generate the index RST file 420 generate_main_index_rst(args.output) 421 422 423if __name__ == "__main__": 424 main() 425