1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 if prefix in mappings: 113 prefix = mappings[prefix] 114 return f":ref:`{namespace}-{prefix}-{name}`" 115 116 def rst_header(self) -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 @staticmethod 126 def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 @staticmethod 136 def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 @staticmethod 141 def rst_lineno(lineno: int) -> str: 142 """Return a lineno comment""" 143 return f".. LINENO {lineno}\n" 144 145class YnlDocGenerator: 146 """YAML Netlink specs Parser""" 147 148 fmt = RstFormatters() 149 150 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 151 """Parse 'multicast' group list and return a formatted string""" 152 lines = [] 153 for group in mcast_group: 154 lines.append(self.fmt.rst_bullet(group["name"])) 155 156 return "\n".join(lines) 157 158 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 159 """Parse 'do' section and return a formatted string""" 160 lines = [] 161 if LINE_STR in do_dict: 162 lines.append(self.fmt.rst_lineno(do_dict[LINE_STR])) 163 164 for key in do_dict.keys(): 165 if key == LINE_STR: 166 continue 167 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 168 if key in ['request', 'reply']: 169 lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n") 170 else: 171 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 172 173 return "\n".join(lines) 174 175 def parse_do_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 176 """Parse 'attributes' section""" 177 if "attributes" not in attrs: 178 return "" 179 lines = [self.fmt.rst_fields("attributes", 180 self.fmt.rst_list_inline(attrs["attributes"]), 181 level + 1)] 182 183 return "\n".join(lines) 184 185 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 186 """Parse operations block""" 187 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 188 linkable = ["fixed-header", "attribute-set"] 189 lines = [] 190 191 for operation in operations: 192 if LINE_STR in operation: 193 lines.append(self.fmt.rst_lineno(operation[LINE_STR])) 194 195 lines.append(self.fmt.rst_section(namespace, 'operation', 196 operation["name"])) 197 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 198 199 for key in operation.keys(): 200 if key == LINE_STR: 201 continue 202 203 if key in preprocessed: 204 # Skip the special fields 205 continue 206 value = operation[key] 207 if key in linkable: 208 value = self.fmt.rst_ref(namespace, key, value) 209 lines.append(self.fmt.rst_fields(key, value, 0)) 210 if 'flags' in operation: 211 lines.append(self.fmt.rst_fields('flags', 212 self.fmt.rst_list_inline(operation['flags']))) 213 214 if "do" in operation: 215 lines.append(self.fmt.rst_paragraph(":do:", 0)) 216 lines.append(self.parse_do(operation["do"], 0)) 217 if "dump" in operation: 218 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 219 lines.append(self.parse_do(operation["dump"], 0)) 220 221 # New line after fields 222 lines.append("\n") 223 224 return "\n".join(lines) 225 226 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 227 """Parse a list of entries""" 228 ignored = ["pad"] 229 lines = [] 230 for entry in entries: 231 if isinstance(entry, dict): 232 # entries could be a list or a dictionary 233 field_name = entry.get("name", "") 234 if field_name in ignored: 235 continue 236 type_ = entry.get("type") 237 if type_: 238 field_name += f" ({self.fmt.inline(type_)})" 239 lines.append( 240 self.fmt.rst_fields(field_name, 241 self.fmt.sanitize(entry.get("doc", "")), 242 level) 243 ) 244 elif isinstance(entry, list): 245 lines.append(self.fmt.rst_list_inline(entry, level)) 246 else: 247 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 248 level)) 249 250 lines.append("\n") 251 return "\n".join(lines) 252 253 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 254 """Parse definitions section""" 255 preprocessed = ["name", "entries", "members"] 256 ignored = ["render-max"] # This is not printed 257 lines = [] 258 259 for definition in defs: 260 if LINE_STR in definition: 261 lines.append(self.fmt.rst_lineno(definition[LINE_STR])) 262 263 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 264 for k in definition.keys(): 265 if k == LINE_STR: 266 continue 267 if k in preprocessed + ignored: 268 continue 269 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 270 271 # Field list needs to finish with a new line 272 lines.append("\n") 273 if "entries" in definition: 274 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 275 lines.append(self.parse_entries(definition["entries"], 1)) 276 if "members" in definition: 277 lines.append(self.fmt.rst_paragraph(":members:", 0)) 278 lines.append(self.parse_entries(definition["members"], 1)) 279 280 return "\n".join(lines) 281 282 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 283 """Parse attribute from attribute-set""" 284 preprocessed = ["name", "type"] 285 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 286 ignored = ["checks"] 287 lines = [] 288 289 for entry in entries: 290 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 291 entry["name"])) 292 for attr in entry["attributes"]: 293 if LINE_STR in attr: 294 lines.append(self.fmt.rst_lineno(attr[LINE_STR])) 295 296 type_ = attr.get("type") 297 attr_line = attr["name"] 298 if type_: 299 # Add the attribute type in the same line 300 attr_line += f" ({self.fmt.inline(type_)})" 301 302 lines.append(self.fmt.rst_subsubsection(attr_line)) 303 304 for k in attr.keys(): 305 if k == LINE_STR: 306 continue 307 if k in preprocessed + ignored: 308 continue 309 if k in linkable: 310 value = self.fmt.rst_ref(namespace, k, attr[k]) 311 else: 312 value = self.fmt.sanitize(attr[k]) 313 lines.append(self.fmt.rst_fields(k, value, 0)) 314 lines.append("\n") 315 316 return "\n".join(lines) 317 318 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 319 """Parse sub-message definitions""" 320 lines = [] 321 322 for entry in entries: 323 lines.append(self.fmt.rst_section(namespace, 'sub-message', 324 entry["name"])) 325 for fmt in entry["formats"]: 326 value = fmt["value"] 327 328 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 329 for attr in ['fixed-header', 'attribute-set']: 330 if attr in fmt: 331 lines.append(self.fmt.rst_fields(attr, 332 self.fmt.rst_ref(namespace, 333 attr, 334 fmt[attr]), 335 1)) 336 lines.append("\n") 337 338 return "\n".join(lines) 339 340 def parse_yaml(self, obj: Dict[str, Any]) -> str: 341 """Format the whole YAML into a RST string""" 342 lines = [] 343 344 # Main header 345 lineno = obj.get('__lineno__', 0) 346 lines.append(self.fmt.rst_lineno(lineno)) 347 348 family = obj['name'] 349 350 lines.append(self.fmt.rst_header()) 351 lines.append(self.fmt.rst_label("netlink-" + family)) 352 353 title = f"Family ``{family}`` netlink specification" 354 lines.append(self.fmt.rst_title(title)) 355 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 356 357 if "doc" in obj: 358 lines.append(self.fmt.rst_subtitle("Summary")) 359 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 360 361 # Operations 362 if "operations" in obj: 363 lines.append(self.fmt.rst_subtitle("Operations")) 364 lines.append(self.parse_operations(obj["operations"]["list"], 365 family)) 366 367 # Multicast groups 368 if "mcast-groups" in obj: 369 lines.append(self.fmt.rst_subtitle("Multicast groups")) 370 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 371 372 # Definitions 373 if "definitions" in obj: 374 lines.append(self.fmt.rst_subtitle("Definitions")) 375 lines.append(self.parse_definitions(obj["definitions"], family)) 376 377 # Attributes set 378 if "attribute-sets" in obj: 379 lines.append(self.fmt.rst_subtitle("Attribute sets")) 380 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 381 382 # Sub-messages 383 if "sub-messages" in obj: 384 lines.append(self.fmt.rst_subtitle("Sub-messages")) 385 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 386 387 return "\n".join(lines) 388 389 # Main functions 390 # ============== 391 392 def parse_yaml_file(self, filename: str) -> str: 393 """Transform the YAML specified by filename into an RST-formatted string""" 394 with open(filename, "r", encoding="utf-8") as spec_file: 395 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 396 content = self.parse_yaml(numbered_yaml) 397 398 return content 399