1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 prefix = mappings.get(prefix, prefix) 113 return f":ref:`{namespace}-{prefix}-{name}`" 114 115 def rst_header(self) -> str: 116 """The headers for all the auto generated RST files""" 117 lines = [] 118 119 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 120 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 121 122 return "\n".join(lines) 123 124 @staticmethod 125 def rst_toctree(maxdepth: int = 2) -> str: 126 """Generate a toctree RST primitive""" 127 lines = [] 128 129 lines.append(".. toctree::") 130 lines.append(f" :maxdepth: {maxdepth}\n\n") 131 132 return "\n".join(lines) 133 134 @staticmethod 135 def rst_label(title: str) -> str: 136 """Return a formatted label""" 137 return f".. _{title}:\n\n" 138 139 @staticmethod 140 def rst_lineno(lineno: int) -> str: 141 """Return a lineno comment""" 142 return f".. LINENO {lineno}\n" 143 144class YnlDocGenerator: 145 """YAML Netlink specs Parser""" 146 147 fmt = RstFormatters() 148 149 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 150 """Parse 'multicast' group list and return a formatted string""" 151 lines = [] 152 for group in mcast_group: 153 lines.append(self.fmt.rst_bullet(group["name"])) 154 155 return "\n".join(lines) 156 157 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 158 """Parse 'do' section and return a formatted string""" 159 lines = [] 160 if LINE_STR in do_dict: 161 lines.append(self.fmt.rst_lineno(do_dict[LINE_STR])) 162 163 for key in do_dict.keys(): 164 if key == LINE_STR: 165 continue 166 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 167 if key in ['request', 'reply']: 168 lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n") 169 else: 170 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 171 172 return "\n".join(lines) 173 174 def parse_do_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 175 """Parse 'attributes' section""" 176 if "attributes" not in attrs: 177 return "" 178 lines = [self.fmt.rst_fields("attributes", 179 self.fmt.rst_list_inline(attrs["attributes"]), 180 level + 1)] 181 182 return "\n".join(lines) 183 184 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 185 """Parse operations block""" 186 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 187 linkable = ["fixed-header", "attribute-set"] 188 lines = [] 189 190 for operation in operations: 191 if LINE_STR in operation: 192 lines.append(self.fmt.rst_lineno(operation[LINE_STR])) 193 194 lines.append(self.fmt.rst_section(namespace, 'operation', 195 operation["name"])) 196 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 197 198 for key in operation.keys(): 199 if key == LINE_STR: 200 continue 201 202 if key in preprocessed: 203 # Skip the special fields 204 continue 205 value = operation[key] 206 if key in linkable: 207 value = self.fmt.rst_ref(namespace, key, value) 208 lines.append(self.fmt.rst_fields(key, value, 0)) 209 if 'flags' in operation: 210 lines.append(self.fmt.rst_fields('flags', 211 self.fmt.rst_list_inline(operation['flags']))) 212 213 if "do" in operation: 214 lines.append(self.fmt.rst_paragraph(":do:", 0)) 215 lines.append(self.parse_do(operation["do"], 0)) 216 if "dump" in operation: 217 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 218 lines.append(self.parse_do(operation["dump"], 0)) 219 220 # New line after fields 221 lines.append("\n") 222 223 return "\n".join(lines) 224 225 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 226 """Parse a list of entries""" 227 ignored = ["pad"] 228 lines = [] 229 for entry in entries: 230 if isinstance(entry, dict): 231 # entries could be a list or a dictionary 232 field_name = entry.get("name", "") 233 if field_name in ignored: 234 continue 235 type_ = entry.get("type") 236 if type_: 237 field_name += f" ({self.fmt.inline(type_)})" 238 lines.append( 239 self.fmt.rst_fields(field_name, 240 self.fmt.sanitize(entry.get("doc", "")), 241 level) 242 ) 243 elif isinstance(entry, list): 244 lines.append(self.fmt.rst_list_inline(entry, level)) 245 else: 246 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 247 level)) 248 249 lines.append("\n") 250 return "\n".join(lines) 251 252 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 253 """Parse definitions section""" 254 preprocessed = ["name", "entries", "members"] 255 ignored = ["render-max"] # This is not printed 256 lines = [] 257 258 for definition in defs: 259 if LINE_STR in definition: 260 lines.append(self.fmt.rst_lineno(definition[LINE_STR])) 261 262 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 263 for k in definition.keys(): 264 if k == LINE_STR: 265 continue 266 if k in preprocessed + ignored: 267 continue 268 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 269 270 # Field list needs to finish with a new line 271 lines.append("\n") 272 if "entries" in definition: 273 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 274 lines.append(self.parse_entries(definition["entries"], 1)) 275 if "members" in definition: 276 lines.append(self.fmt.rst_paragraph(":members:", 0)) 277 lines.append(self.parse_entries(definition["members"], 1)) 278 279 return "\n".join(lines) 280 281 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 282 """Parse attribute from attribute-set""" 283 preprocessed = ["name", "type"] 284 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 285 ignored = ["checks"] 286 lines = [] 287 288 for entry in entries: 289 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 290 entry["name"])) 291 292 if "doc" in entry: 293 lines.append(self.fmt.rst_paragraph(entry["doc"], 0) + "\n") 294 295 for attr in entry["attributes"]: 296 if LINE_STR in attr: 297 lines.append(self.fmt.rst_lineno(attr[LINE_STR])) 298 299 type_ = attr.get("type") 300 attr_line = attr["name"] 301 if type_: 302 # Add the attribute type in the same line 303 attr_line += f" ({self.fmt.inline(type_)})" 304 305 lines.append(self.fmt.rst_subsubsection(attr_line)) 306 307 for k in attr.keys(): 308 if k == LINE_STR: 309 continue 310 if k in preprocessed + ignored: 311 continue 312 if k in linkable: 313 value = self.fmt.rst_ref(namespace, k, attr[k]) 314 else: 315 value = self.fmt.sanitize(attr[k]) 316 lines.append(self.fmt.rst_fields(k, value, 0)) 317 lines.append("\n") 318 319 return "\n".join(lines) 320 321 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 322 """Parse sub-message definitions""" 323 lines = [] 324 325 for entry in entries: 326 lines.append(self.fmt.rst_section(namespace, 'sub-message', 327 entry["name"])) 328 for fmt in entry["formats"]: 329 value = fmt["value"] 330 331 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 332 for attr in ['fixed-header', 'attribute-set']: 333 if attr in fmt: 334 lines.append(self.fmt.rst_fields(attr, 335 self.fmt.rst_ref(namespace, 336 attr, 337 fmt[attr]), 338 1)) 339 lines.append("\n") 340 341 return "\n".join(lines) 342 343 def parse_yaml(self, obj: Dict[str, Any]) -> str: 344 """Format the whole YAML into a RST string""" 345 lines = [] 346 347 # Main header 348 lineno = obj.get('__lineno__', 0) 349 lines.append(self.fmt.rst_lineno(lineno)) 350 351 family = obj['name'] 352 353 lines.append(self.fmt.rst_header()) 354 lines.append(self.fmt.rst_label("netlink-" + family)) 355 356 title = f"Family ``{family}`` netlink specification" 357 lines.append(self.fmt.rst_title(title)) 358 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 359 360 if "doc" in obj: 361 lines.append(self.fmt.rst_subtitle("Summary")) 362 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 363 364 # Operations 365 if "operations" in obj: 366 lines.append(self.fmt.rst_subtitle("Operations")) 367 lines.append(self.parse_operations(obj["operations"]["list"], 368 family)) 369 370 # Multicast groups 371 if "mcast-groups" in obj: 372 lines.append(self.fmt.rst_subtitle("Multicast groups")) 373 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 374 375 # Definitions 376 if "definitions" in obj: 377 lines.append(self.fmt.rst_subtitle("Definitions")) 378 lines.append(self.parse_definitions(obj["definitions"], family)) 379 380 # Attributes set 381 if "attribute-sets" in obj: 382 lines.append(self.fmt.rst_subtitle("Attribute sets")) 383 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 384 385 # Sub-messages 386 if "sub-messages" in obj: 387 lines.append(self.fmt.rst_subtitle("Sub-messages")) 388 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 389 390 return "\n".join(lines) 391 392 # Main functions 393 # ============== 394 395 def parse_yaml_file(self, filename: str) -> str: 396 """Transform the YAML specified by filename into an RST-formatted string""" 397 with open(filename, "r", encoding="utf-8") as spec_file: 398 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 399 content = self.parse_yaml(numbered_yaml) 400 401 return content 402