1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Script to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This script performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in three big parts: 16 1) RST formatters: Use to convert a string to a RST output 17 2) Parser helpers: Functions to parse the YAML data structure 18 3) Main function and small helpers 19""" 20 21from typing import Any, Dict, List 22import os.path 23import sys 24import argparse 25import logging 26import yaml 27 28 29SPACE_PER_LEVEL = 4 30 31 32# RST Formatters 33# ============== 34def headroom(level: int) -> str: 35 """Return space to format""" 36 return " " * (level * SPACE_PER_LEVEL) 37 38 39def bold(text: str) -> str: 40 """Format bold text""" 41 return f"**{text}**" 42 43 44def inline(text: str) -> str: 45 """Format inline text""" 46 return f"``{text}``" 47 48 49def sanitize(text: str) -> str: 50 """Remove newlines and multiple spaces""" 51 # This is useful for some fields that are spread across multiple lines 52 return str(text).replace("\n", "").strip() 53 54 55def rst_fields(key: str, value: str, level: int = 0) -> str: 56 """Return a RST formatted field""" 57 return headroom(level) + f":{key}: {value}" 58 59 60def rst_definition(key: str, value: Any, level: int = 0) -> str: 61 """Format a single rst definition""" 62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value) 63 64 65def rst_paragraph(paragraph: str, level: int = 0) -> str: 66 """Return a formatted paragraph""" 67 return headroom(level) + paragraph 68 69 70def rst_bullet(item: str, level: int = 0) -> str: 71 """Return a formatted a bullet""" 72 return headroom(level) + f"- {item}" 73 74 75def rst_subsection(title: str) -> str: 76 """Add a sub-section to the document""" 77 return f"{title}\n" + "-" * len(title) 78 79 80def rst_subsubsection(title: str) -> str: 81 """Add a sub-sub-section to the document""" 82 return f"{title}\n" + "~" * len(title) 83 84 85def rst_section(title: str) -> str: 86 """Add a section to the document""" 87 return f"\n{title}\n" + "=" * len(title) 88 89 90def rst_subtitle(title: str) -> str: 91 """Add a subtitle to the document""" 92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 93 94 95def rst_title(title: str) -> str: 96 """Add a title to the document""" 97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 98 99 100def rst_list_inline(list_: List[str], level: int = 0) -> str: 101 """Format a list using inlines""" 102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]" 103 104 105def rst_header() -> str: 106 """The headers for all the auto generated RST files""" 107 lines = [] 108 109 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 110 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 111 112 return "\n".join(lines) 113 114 115def rst_toctree(maxdepth: int = 2) -> str: 116 """Generate a toctree RST primitive""" 117 lines = [] 118 119 lines.append(".. toctree::") 120 lines.append(f" :maxdepth: {maxdepth}\n\n") 121 122 return "\n".join(lines) 123 124 125def rst_label(title: str) -> str: 126 """Return a formatted label""" 127 return f".. _{title}:\n\n" 128 129 130# Parsers 131# ======= 132 133 134def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str: 135 """Parse 'multicast' group list and return a formatted string""" 136 lines = [] 137 for group in mcast_group: 138 lines.append(rst_bullet(group["name"])) 139 140 return "\n".join(lines) 141 142 143def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str: 144 """Parse 'do' section and return a formatted string""" 145 lines = [] 146 for key in do_dict.keys(): 147 lines.append(rst_paragraph(bold(key), level + 1)) 148 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n") 149 150 return "\n".join(lines) 151 152 153def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str: 154 """Parse 'attributes' section""" 155 if "attributes" not in attrs: 156 return "" 157 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)] 158 159 return "\n".join(lines) 160 161 162def parse_operations(operations: List[Dict[str, Any]]) -> str: 163 """Parse operations block""" 164 preprocessed = ["name", "doc", "title", "do", "dump"] 165 lines = [] 166 167 for operation in operations: 168 lines.append(rst_section(operation["name"])) 169 lines.append(rst_paragraph(sanitize(operation["doc"])) + "\n") 170 171 for key in operation.keys(): 172 if key in preprocessed: 173 # Skip the special fields 174 continue 175 lines.append(rst_fields(key, operation[key], 0)) 176 177 if "do" in operation: 178 lines.append(rst_paragraph(":do:", 0)) 179 lines.append(parse_do(operation["do"], 0)) 180 if "dump" in operation: 181 lines.append(rst_paragraph(":dump:", 0)) 182 lines.append(parse_do(operation["dump"], 0)) 183 184 # New line after fields 185 lines.append("\n") 186 187 return "\n".join(lines) 188 189 190def parse_entries(entries: List[Dict[str, Any]], level: int) -> str: 191 """Parse a list of entries""" 192 lines = [] 193 for entry in entries: 194 if isinstance(entry, dict): 195 # entries could be a list or a dictionary 196 lines.append( 197 rst_fields(entry.get("name", ""), sanitize(entry.get("doc", "")), level) 198 ) 199 elif isinstance(entry, list): 200 lines.append(rst_list_inline(entry, level)) 201 else: 202 lines.append(rst_bullet(inline(sanitize(entry)), level)) 203 204 lines.append("\n") 205 return "\n".join(lines) 206 207 208def parse_definitions(defs: Dict[str, Any]) -> str: 209 """Parse definitions section""" 210 preprocessed = ["name", "entries", "members"] 211 ignored = ["render-max"] # This is not printed 212 lines = [] 213 214 for definition in defs: 215 lines.append(rst_section(definition["name"])) 216 for k in definition.keys(): 217 if k in preprocessed + ignored: 218 continue 219 lines.append(rst_fields(k, sanitize(definition[k]), 0)) 220 221 # Field list needs to finish with a new line 222 lines.append("\n") 223 if "entries" in definition: 224 lines.append(rst_paragraph(":entries:", 0)) 225 lines.append(parse_entries(definition["entries"], 1)) 226 if "members" in definition: 227 lines.append(rst_paragraph(":members:", 0)) 228 lines.append(parse_entries(definition["members"], 1)) 229 230 return "\n".join(lines) 231 232 233def parse_attr_sets(entries: List[Dict[str, Any]]) -> str: 234 """Parse attribute from attribute-set""" 235 preprocessed = ["name", "type"] 236 ignored = ["checks"] 237 lines = [] 238 239 for entry in entries: 240 lines.append(rst_section(entry["name"])) 241 for attr in entry["attributes"]: 242 type_ = attr.get("type") 243 attr_line = attr["name"] 244 if type_: 245 # Add the attribute type in the same line 246 attr_line += f" ({inline(type_)})" 247 248 lines.append(rst_subsubsection(attr_line)) 249 250 for k in attr.keys(): 251 if k in preprocessed + ignored: 252 continue 253 lines.append(rst_fields(k, sanitize(attr[k]), 0)) 254 lines.append("\n") 255 256 return "\n".join(lines) 257 258 259def parse_sub_messages(entries: List[Dict[str, Any]]) -> str: 260 """Parse sub-message definitions""" 261 lines = [] 262 263 for entry in entries: 264 lines.append(rst_section(entry["name"])) 265 for fmt in entry["formats"]: 266 value = fmt["value"] 267 268 lines.append(rst_bullet(bold(value))) 269 for attr in ['fixed-header', 'attribute-set']: 270 if attr in fmt: 271 lines.append(rst_fields(attr, fmt[attr], 1)) 272 lines.append("\n") 273 274 return "\n".join(lines) 275 276 277def parse_yaml(obj: Dict[str, Any]) -> str: 278 """Format the whole YAML into a RST string""" 279 lines = [] 280 281 # Main header 282 283 lines.append(rst_header()) 284 285 title = f"Family ``{obj['name']}`` netlink specification" 286 lines.append(rst_title(title)) 287 lines.append(rst_paragraph(".. contents::\n")) 288 289 if "doc" in obj: 290 lines.append(rst_subtitle("Summary")) 291 lines.append(rst_paragraph(obj["doc"], 0)) 292 293 # Operations 294 if "operations" in obj: 295 lines.append(rst_subtitle("Operations")) 296 lines.append(parse_operations(obj["operations"]["list"])) 297 298 # Multicast groups 299 if "mcast-groups" in obj: 300 lines.append(rst_subtitle("Multicast groups")) 301 lines.append(parse_mcast_group(obj["mcast-groups"]["list"])) 302 303 # Definitions 304 if "definitions" in obj: 305 lines.append(rst_subtitle("Definitions")) 306 lines.append(parse_definitions(obj["definitions"])) 307 308 # Attributes set 309 if "attribute-sets" in obj: 310 lines.append(rst_subtitle("Attribute sets")) 311 lines.append(parse_attr_sets(obj["attribute-sets"])) 312 313 # Sub-messages 314 if "sub-messages" in obj: 315 lines.append(rst_subtitle("Sub-messages")) 316 lines.append(parse_sub_messages(obj["sub-messages"])) 317 318 return "\n".join(lines) 319 320 321# Main functions 322# ============== 323 324 325def parse_arguments() -> argparse.Namespace: 326 """Parse arguments from user""" 327 parser = argparse.ArgumentParser(description="Netlink RST generator") 328 329 parser.add_argument("-v", "--verbose", action="store_true") 330 parser.add_argument("-o", "--output", help="Output file name") 331 332 # Index and input are mutually exclusive 333 group = parser.add_mutually_exclusive_group() 334 group.add_argument( 335 "-x", "--index", action="store_true", help="Generate the index page" 336 ) 337 group.add_argument("-i", "--input", help="YAML file name") 338 339 args = parser.parse_args() 340 341 if args.verbose: 342 logging.basicConfig(level=logging.DEBUG) 343 344 if args.input and not os.path.isfile(args.input): 345 logging.warning("%s is not a valid file.", args.input) 346 sys.exit(-1) 347 348 if not args.output: 349 logging.error("No output file specified.") 350 sys.exit(-1) 351 352 if os.path.isfile(args.output): 353 logging.debug("%s already exists. Overwriting it.", args.output) 354 355 return args 356 357 358def parse_yaml_file(filename: str) -> str: 359 """Transform the YAML specified by filename into a rst-formmated string""" 360 with open(filename, "r", encoding="utf-8") as spec_file: 361 yaml_data = yaml.safe_load(spec_file) 362 content = parse_yaml(yaml_data) 363 364 return content 365 366 367def write_to_rstfile(content: str, filename: str) -> None: 368 """Write the generated content into an RST file""" 369 logging.debug("Saving RST file to %s", filename) 370 371 with open(filename, "w", encoding="utf-8") as rst_file: 372 rst_file.write(content) 373 374 375def generate_main_index_rst(output: str) -> None: 376 """Generate the `networking_spec/index` content and write to the file""" 377 lines = [] 378 379 lines.append(rst_header()) 380 lines.append(rst_label("specs")) 381 lines.append(rst_title("Netlink Family Specifications")) 382 lines.append(rst_toctree(1)) 383 384 index_dir = os.path.dirname(output) 385 logging.debug("Looking for .rst files in %s", index_dir) 386 for filename in sorted(os.listdir(index_dir)): 387 if not filename.endswith(".rst") or filename == "index.rst": 388 continue 389 lines.append(f" {filename.replace('.rst', '')}\n") 390 391 logging.debug("Writing an index file at %s", output) 392 write_to_rstfile("".join(lines), output) 393 394 395def main() -> None: 396 """Main function that reads the YAML files and generates the RST files""" 397 398 args = parse_arguments() 399 400 if args.input: 401 logging.debug("Parsing %s", args.input) 402 try: 403 content = parse_yaml_file(os.path.join(args.input)) 404 except Exception as exception: 405 logging.warning("Failed to parse %s.", args.input) 406 logging.warning(exception) 407 sys.exit(-1) 408 409 write_to_rstfile(content, args.output) 410 411 if args.index: 412 # Generate the index RST file 413 generate_main_index_rst(args.output) 414 415 416if __name__ == "__main__": 417 main() 418