1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 if prefix in mappings: 113 prefix = mappings[prefix] 114 return f":ref:`{namespace}-{prefix}-{name}`" 115 116 def rst_header(self) -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 @staticmethod 126 def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 @staticmethod 136 def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 @staticmethod 141 def rst_lineno(lineno: int) -> str: 142 """Return a lineno comment""" 143 return f".. LINENO {lineno}\n" 144 145class YnlDocGenerator: 146 """YAML Netlink specs Parser""" 147 148 fmt = RstFormatters() 149 150 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 151 """Parse 'multicast' group list and return a formatted string""" 152 lines = [] 153 for group in mcast_group: 154 lines.append(self.fmt.rst_bullet(group["name"])) 155 156 return "\n".join(lines) 157 158 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 159 """Parse 'do' section and return a formatted string""" 160 lines = [] 161 if LINE_STR in do_dict: 162 lines.append(self.fmt.rst_lineno(do_dict[LINE_STR])) 163 164 for key in do_dict.keys(): 165 if key == LINE_STR: 166 continue 167 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 168 if key in ['request', 'reply']: 169 lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n") 170 else: 171 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 172 173 return "\n".join(lines) 174 175 def parse_do_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 176 """Parse 'attributes' section""" 177 if "attributes" not in attrs: 178 return "" 179 lines = [self.fmt.rst_fields("attributes", 180 self.fmt.rst_list_inline(attrs["attributes"]), 181 level + 1)] 182 183 return "\n".join(lines) 184 185 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 186 """Parse operations block""" 187 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 188 linkable = ["fixed-header", "attribute-set"] 189 lines = [] 190 191 for operation in operations: 192 if LINE_STR in operation: 193 lines.append(self.fmt.rst_lineno(operation[LINE_STR])) 194 195 lines.append(self.fmt.rst_section(namespace, 'operation', 196 operation["name"])) 197 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 198 199 for key in operation.keys(): 200 if key == LINE_STR: 201 continue 202 203 if key in preprocessed: 204 # Skip the special fields 205 continue 206 value = operation[key] 207 if key in linkable: 208 value = self.fmt.rst_ref(namespace, key, value) 209 lines.append(self.fmt.rst_fields(key, value, 0)) 210 if 'flags' in operation: 211 lines.append(self.fmt.rst_fields('flags', 212 self.fmt.rst_list_inline(operation['flags']))) 213 214 if "do" in operation: 215 lines.append(self.fmt.rst_paragraph(":do:", 0)) 216 lines.append(self.parse_do(operation["do"], 0)) 217 if "dump" in operation: 218 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 219 lines.append(self.parse_do(operation["dump"], 0)) 220 221 # New line after fields 222 lines.append("\n") 223 224 return "\n".join(lines) 225 226 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 227 """Parse a list of entries""" 228 ignored = ["pad"] 229 lines = [] 230 for entry in entries: 231 if isinstance(entry, dict): 232 # entries could be a list or a dictionary 233 field_name = entry.get("name", "") 234 if field_name in ignored: 235 continue 236 type_ = entry.get("type") 237 if type_: 238 field_name += f" ({self.fmt.inline(type_)})" 239 lines.append( 240 self.fmt.rst_fields(field_name, 241 self.fmt.sanitize(entry.get("doc", "")), 242 level) 243 ) 244 elif isinstance(entry, list): 245 lines.append(self.fmt.rst_list_inline(entry, level)) 246 else: 247 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 248 level)) 249 250 lines.append("\n") 251 return "\n".join(lines) 252 253 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 254 """Parse definitions section""" 255 preprocessed = ["name", "entries", "members"] 256 ignored = ["render-max"] # This is not printed 257 lines = [] 258 259 for definition in defs: 260 if LINE_STR in definition: 261 lines.append(self.fmt.rst_lineno(definition[LINE_STR])) 262 263 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 264 for k in definition.keys(): 265 if k == LINE_STR: 266 continue 267 if k in preprocessed + ignored: 268 continue 269 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 270 271 # Field list needs to finish with a new line 272 lines.append("\n") 273 if "entries" in definition: 274 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 275 lines.append(self.parse_entries(definition["entries"], 1)) 276 if "members" in definition: 277 lines.append(self.fmt.rst_paragraph(":members:", 0)) 278 lines.append(self.parse_entries(definition["members"], 1)) 279 280 return "\n".join(lines) 281 282 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 283 """Parse attribute from attribute-set""" 284 preprocessed = ["name", "type"] 285 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 286 ignored = ["checks"] 287 lines = [] 288 289 for entry in entries: 290 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 291 entry["name"])) 292 293 if "doc" in entry: 294 lines.append(self.fmt.rst_paragraph(entry["doc"], 0) + "\n") 295 296 for attr in entry["attributes"]: 297 if LINE_STR in attr: 298 lines.append(self.fmt.rst_lineno(attr[LINE_STR])) 299 300 type_ = attr.get("type") 301 attr_line = attr["name"] 302 if type_: 303 # Add the attribute type in the same line 304 attr_line += f" ({self.fmt.inline(type_)})" 305 306 lines.append(self.fmt.rst_subsubsection(attr_line)) 307 308 for k in attr.keys(): 309 if k == LINE_STR: 310 continue 311 if k in preprocessed + ignored: 312 continue 313 if k in linkable: 314 value = self.fmt.rst_ref(namespace, k, attr[k]) 315 else: 316 value = self.fmt.sanitize(attr[k]) 317 lines.append(self.fmt.rst_fields(k, value, 0)) 318 lines.append("\n") 319 320 return "\n".join(lines) 321 322 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 323 """Parse sub-message definitions""" 324 lines = [] 325 326 for entry in entries: 327 lines.append(self.fmt.rst_section(namespace, 'sub-message', 328 entry["name"])) 329 for fmt in entry["formats"]: 330 value = fmt["value"] 331 332 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 333 for attr in ['fixed-header', 'attribute-set']: 334 if attr in fmt: 335 lines.append(self.fmt.rst_fields(attr, 336 self.fmt.rst_ref(namespace, 337 attr, 338 fmt[attr]), 339 1)) 340 lines.append("\n") 341 342 return "\n".join(lines) 343 344 def parse_yaml(self, obj: Dict[str, Any]) -> str: 345 """Format the whole YAML into a RST string""" 346 lines = [] 347 348 # Main header 349 lineno = obj.get('__lineno__', 0) 350 lines.append(self.fmt.rst_lineno(lineno)) 351 352 family = obj['name'] 353 354 lines.append(self.fmt.rst_header()) 355 lines.append(self.fmt.rst_label("netlink-" + family)) 356 357 title = f"Family ``{family}`` netlink specification" 358 lines.append(self.fmt.rst_title(title)) 359 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 360 361 if "doc" in obj: 362 lines.append(self.fmt.rst_subtitle("Summary")) 363 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 364 365 # Operations 366 if "operations" in obj: 367 lines.append(self.fmt.rst_subtitle("Operations")) 368 lines.append(self.parse_operations(obj["operations"]["list"], 369 family)) 370 371 # Multicast groups 372 if "mcast-groups" in obj: 373 lines.append(self.fmt.rst_subtitle("Multicast groups")) 374 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 375 376 # Definitions 377 if "definitions" in obj: 378 lines.append(self.fmt.rst_subtitle("Definitions")) 379 lines.append(self.parse_definitions(obj["definitions"], family)) 380 381 # Attributes set 382 if "attribute-sets" in obj: 383 lines.append(self.fmt.rst_subtitle("Attribute sets")) 384 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 385 386 # Sub-messages 387 if "sub-messages" in obj: 388 lines.append(self.fmt.rst_subtitle("Sub-messages")) 389 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 390 391 return "\n".join(lines) 392 393 # Main functions 394 # ============== 395 396 def parse_yaml_file(self, filename: str) -> str: 397 """Transform the YAML specified by filename into an RST-formatted string""" 398 with open(filename, "r", encoding="utf-8") as spec_file: 399 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 400 content = self.parse_yaml(numbered_yaml) 401 402 return content 403