1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Script to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This script performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in three big parts: 16 1) RST formatters: Use to convert a string to a RST output 17 2) Parser helpers: Functions to parse the YAML data structure 18 3) Main function and small helpers 19""" 20 21from typing import Any, Dict, List 22import os.path 23import sys 24import argparse 25import logging 26import yaml 27 28 29SPACE_PER_LEVEL = 4 30 31 32# RST Formatters 33# ============== 34def headroom(level: int) -> str: 35 """Return space to format""" 36 return " " * (level * SPACE_PER_LEVEL) 37 38 39def bold(text: str) -> str: 40 """Format bold text""" 41 return f"**{text}**" 42 43 44def inline(text: str) -> str: 45 """Format inline text""" 46 return f"``{text}``" 47 48 49def sanitize(text: str) -> str: 50 """Remove newlines and multiple spaces""" 51 # This is useful for some fields that are spread across multiple lines 52 return str(text).replace("\n", "").strip() 53 54 55def rst_fields(key: str, value: str, level: int = 0) -> str: 56 """Return a RST formatted field""" 57 return headroom(level) + f":{key}: {value}" 58 59 60def rst_definition(key: str, value: Any, level: int = 0) -> str: 61 """Format a single rst definition""" 62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value) 63 64 65def rst_paragraph(paragraph: str, level: int = 0) -> str: 66 """Return a formatted paragraph""" 67 return headroom(level) + paragraph 68 69 70def rst_bullet(item: str, level: int = 0) -> str: 71 """Return a formatted a bullet""" 72 return headroom(level) + f"- {item}" 73 74 75def rst_subsection(title: str) -> str: 76 """Add a sub-section to the document""" 77 return f"{title}\n" + "-" * len(title) 78 79 80def rst_subsubsection(title: str) -> str: 81 """Add a sub-sub-section to the document""" 82 return f"{title}\n" + "~" * len(title) 83 84 85def rst_section(namespace: str, prefix: str, title: str) -> str: 86 """Add a section to the document""" 87 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 88 89 90def rst_subtitle(title: str) -> str: 91 """Add a subtitle to the document""" 92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 93 94 95def rst_title(title: str) -> str: 96 """Add a title to the document""" 97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 98 99 100def rst_list_inline(list_: List[str], level: int = 0) -> str: 101 """Format a list using inlines""" 102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]" 103 104 105def rst_ref(namespace: str, prefix: str, name: str) -> str: 106 """Add a hyperlink to the document""" 107 mappings = {'enum': 'definition', 108 'fixed-header': 'definition', 109 'nested-attributes': 'attribute-set', 110 'struct': 'definition'} 111 if prefix in mappings: 112 prefix = mappings[prefix] 113 return f":ref:`{namespace}-{prefix}-{name}`" 114 115 116def rst_header() -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 126def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 136def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 141# Parsers 142# ======= 143 144 145def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str: 146 """Parse 'multicast' group list and return a formatted string""" 147 lines = [] 148 for group in mcast_group: 149 lines.append(rst_bullet(group["name"])) 150 151 return "\n".join(lines) 152 153 154def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str: 155 """Parse 'do' section and return a formatted string""" 156 lines = [] 157 for key in do_dict.keys(): 158 lines.append(rst_paragraph(bold(key), level + 1)) 159 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n") 160 161 return "\n".join(lines) 162 163 164def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str: 165 """Parse 'attributes' section""" 166 if "attributes" not in attrs: 167 return "" 168 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)] 169 170 return "\n".join(lines) 171 172 173def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str: 174 """Parse operations block""" 175 preprocessed = ["name", "doc", "title", "do", "dump"] 176 linkable = ["fixed-header", "attribute-set"] 177 lines = [] 178 179 for operation in operations: 180 lines.append(rst_section(namespace, 'operation', operation["name"])) 181 lines.append(rst_paragraph(sanitize(operation["doc"])) + "\n") 182 183 for key in operation.keys(): 184 if key in preprocessed: 185 # Skip the special fields 186 continue 187 value = operation[key] 188 if key in linkable: 189 value = rst_ref(namespace, key, value) 190 lines.append(rst_fields(key, value, 0)) 191 192 if "do" in operation: 193 lines.append(rst_paragraph(":do:", 0)) 194 lines.append(parse_do(operation["do"], 0)) 195 if "dump" in operation: 196 lines.append(rst_paragraph(":dump:", 0)) 197 lines.append(parse_do(operation["dump"], 0)) 198 199 # New line after fields 200 lines.append("\n") 201 202 return "\n".join(lines) 203 204 205def parse_entries(entries: List[Dict[str, Any]], level: int) -> str: 206 """Parse a list of entries""" 207 ignored = ["pad"] 208 lines = [] 209 for entry in entries: 210 if isinstance(entry, dict): 211 # entries could be a list or a dictionary 212 field_name = entry.get("name", "") 213 if field_name in ignored: 214 continue 215 type_ = entry.get("type") 216 if type_: 217 field_name += f" ({inline(type_)})" 218 lines.append( 219 rst_fields(field_name, sanitize(entry.get("doc", "")), level) 220 ) 221 elif isinstance(entry, list): 222 lines.append(rst_list_inline(entry, level)) 223 else: 224 lines.append(rst_bullet(inline(sanitize(entry)), level)) 225 226 lines.append("\n") 227 return "\n".join(lines) 228 229 230def parse_definitions(defs: Dict[str, Any], namespace: str) -> str: 231 """Parse definitions section""" 232 preprocessed = ["name", "entries", "members"] 233 ignored = ["render-max"] # This is not printed 234 lines = [] 235 236 for definition in defs: 237 lines.append(rst_section(namespace, 'definition', definition["name"])) 238 for k in definition.keys(): 239 if k in preprocessed + ignored: 240 continue 241 lines.append(rst_fields(k, sanitize(definition[k]), 0)) 242 243 # Field list needs to finish with a new line 244 lines.append("\n") 245 if "entries" in definition: 246 lines.append(rst_paragraph(":entries:", 0)) 247 lines.append(parse_entries(definition["entries"], 1)) 248 if "members" in definition: 249 lines.append(rst_paragraph(":members:", 0)) 250 lines.append(parse_entries(definition["members"], 1)) 251 252 return "\n".join(lines) 253 254 255def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str: 256 """Parse attribute from attribute-set""" 257 preprocessed = ["name", "type"] 258 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 259 ignored = ["checks"] 260 lines = [] 261 262 for entry in entries: 263 lines.append(rst_section(namespace, 'attribute-set', entry["name"])) 264 for attr in entry["attributes"]: 265 type_ = attr.get("type") 266 attr_line = attr["name"] 267 if type_: 268 # Add the attribute type in the same line 269 attr_line += f" ({inline(type_)})" 270 271 lines.append(rst_subsubsection(attr_line)) 272 273 for k in attr.keys(): 274 if k in preprocessed + ignored: 275 continue 276 if k in linkable: 277 value = rst_ref(namespace, k, attr[k]) 278 else: 279 value = sanitize(attr[k]) 280 lines.append(rst_fields(k, value, 0)) 281 lines.append("\n") 282 283 return "\n".join(lines) 284 285 286def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str: 287 """Parse sub-message definitions""" 288 lines = [] 289 290 for entry in entries: 291 lines.append(rst_section(namespace, 'sub-message', entry["name"])) 292 for fmt in entry["formats"]: 293 value = fmt["value"] 294 295 lines.append(rst_bullet(bold(value))) 296 for attr in ['fixed-header', 'attribute-set']: 297 if attr in fmt: 298 lines.append(rst_fields(attr, 299 rst_ref(namespace, attr, fmt[attr]), 300 1)) 301 lines.append("\n") 302 303 return "\n".join(lines) 304 305 306def parse_yaml(obj: Dict[str, Any]) -> str: 307 """Format the whole YAML into a RST string""" 308 lines = [] 309 310 # Main header 311 312 lines.append(rst_header()) 313 314 family = obj['name'] 315 316 title = f"Family ``{family}`` netlink specification" 317 lines.append(rst_title(title)) 318 lines.append(rst_paragraph(".. contents:: :depth: 3\n")) 319 320 if "doc" in obj: 321 lines.append(rst_subtitle("Summary")) 322 lines.append(rst_paragraph(obj["doc"], 0)) 323 324 # Operations 325 if "operations" in obj: 326 lines.append(rst_subtitle("Operations")) 327 lines.append(parse_operations(obj["operations"]["list"], family)) 328 329 # Multicast groups 330 if "mcast-groups" in obj: 331 lines.append(rst_subtitle("Multicast groups")) 332 lines.append(parse_mcast_group(obj["mcast-groups"]["list"])) 333 334 # Definitions 335 if "definitions" in obj: 336 lines.append(rst_subtitle("Definitions")) 337 lines.append(parse_definitions(obj["definitions"], family)) 338 339 # Attributes set 340 if "attribute-sets" in obj: 341 lines.append(rst_subtitle("Attribute sets")) 342 lines.append(parse_attr_sets(obj["attribute-sets"], family)) 343 344 # Sub-messages 345 if "sub-messages" in obj: 346 lines.append(rst_subtitle("Sub-messages")) 347 lines.append(parse_sub_messages(obj["sub-messages"], family)) 348 349 return "\n".join(lines) 350 351 352# Main functions 353# ============== 354 355 356def parse_arguments() -> argparse.Namespace: 357 """Parse arguments from user""" 358 parser = argparse.ArgumentParser(description="Netlink RST generator") 359 360 parser.add_argument("-v", "--verbose", action="store_true") 361 parser.add_argument("-o", "--output", help="Output file name") 362 363 # Index and input are mutually exclusive 364 group = parser.add_mutually_exclusive_group() 365 group.add_argument( 366 "-x", "--index", action="store_true", help="Generate the index page" 367 ) 368 group.add_argument("-i", "--input", help="YAML file name") 369 370 args = parser.parse_args() 371 372 if args.verbose: 373 logging.basicConfig(level=logging.DEBUG) 374 375 if args.input and not os.path.isfile(args.input): 376 logging.warning("%s is not a valid file.", args.input) 377 sys.exit(-1) 378 379 if not args.output: 380 logging.error("No output file specified.") 381 sys.exit(-1) 382 383 if os.path.isfile(args.output): 384 logging.debug("%s already exists. Overwriting it.", args.output) 385 386 return args 387 388 389def parse_yaml_file(filename: str) -> str: 390 """Transform the YAML specified by filename into a rst-formmated string""" 391 with open(filename, "r", encoding="utf-8") as spec_file: 392 yaml_data = yaml.safe_load(spec_file) 393 content = parse_yaml(yaml_data) 394 395 return content 396 397 398def write_to_rstfile(content: str, filename: str) -> None: 399 """Write the generated content into an RST file""" 400 logging.debug("Saving RST file to %s", filename) 401 402 with open(filename, "w", encoding="utf-8") as rst_file: 403 rst_file.write(content) 404 405 406def generate_main_index_rst(output: str) -> None: 407 """Generate the `networking_spec/index` content and write to the file""" 408 lines = [] 409 410 lines.append(rst_header()) 411 lines.append(rst_label("specs")) 412 lines.append(rst_title("Netlink Family Specifications")) 413 lines.append(rst_toctree(1)) 414 415 index_dir = os.path.dirname(output) 416 logging.debug("Looking for .rst files in %s", index_dir) 417 for filename in sorted(os.listdir(index_dir)): 418 if not filename.endswith(".rst") or filename == "index.rst": 419 continue 420 lines.append(f" {filename.replace('.rst', '')}\n") 421 422 logging.debug("Writing an index file at %s", output) 423 write_to_rstfile("".join(lines), output) 424 425 426def main() -> None: 427 """Main function that reads the YAML files and generates the RST files""" 428 429 args = parse_arguments() 430 431 if args.input: 432 logging.debug("Parsing %s", args.input) 433 try: 434 content = parse_yaml_file(os.path.join(args.input)) 435 except Exception as exception: 436 logging.warning("Failed to parse %s.", args.input) 437 logging.warning(exception) 438 sys.exit(-1) 439 440 write_to_rstfile(content, args.output) 441 442 if args.index: 443 # Generate the index RST file 444 generate_main_index_rst(args.output) 445 446 447if __name__ == "__main__": 448 main() 449