1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 if prefix in mappings: 113 prefix = mappings[prefix] 114 return f":ref:`{namespace}-{prefix}-{name}`" 115 116 def rst_header(self) -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 @staticmethod 126 def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 @staticmethod 136 def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 @staticmethod 141 def rst_lineno(lineno: int) -> str: 142 """Return a lineno comment""" 143 return f".. LINENO {lineno}\n" 144 145class YnlDocGenerator: 146 """YAML Netlink specs Parser""" 147 148 fmt = RstFormatters() 149 150 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 151 """Parse 'multicast' group list and return a formatted string""" 152 lines = [] 153 for group in mcast_group: 154 lines.append(self.fmt.rst_bullet(group["name"])) 155 156 return "\n".join(lines) 157 158 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 159 """Parse 'do' section and return a formatted string""" 160 lines = [] 161 if LINE_STR in do_dict: 162 lines.append(self.fmt.rst_lineno(do_dict[LINE_STR])) 163 164 for key in do_dict.keys(): 165 if key == LINE_STR: 166 continue 167 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 168 if key in ['request', 'reply']: 169 lines.append(self.parse_op_attributes(do_dict[key], level + 1) + "\n") 170 else: 171 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 172 173 return "\n".join(lines) 174 175 def parse_op_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 176 """Parse 'attributes' section""" 177 if "attributes" not in attrs: 178 return "" 179 lines = [self.fmt.rst_fields("attributes", 180 self.fmt.rst_list_inline(attrs["attributes"]), 181 level + 1)] 182 183 return "\n".join(lines) 184 185 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 186 """Parse operations block""" 187 preprocessed = ["name", "doc", "title", "do", "dump", "flags", "event"] 188 linkable = ["fixed-header", "attribute-set"] 189 lines = [] 190 191 for operation in operations: 192 if LINE_STR in operation: 193 lines.append(self.fmt.rst_lineno(operation[LINE_STR])) 194 195 lines.append(self.fmt.rst_section(namespace, 'operation', 196 operation["name"])) 197 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 198 199 for key in operation.keys(): 200 if key == LINE_STR: 201 continue 202 203 if key in preprocessed: 204 # Skip the special fields 205 continue 206 value = operation[key] 207 if key in linkable: 208 value = self.fmt.rst_ref(namespace, key, value) 209 lines.append(self.fmt.rst_fields(key, value, 0)) 210 if 'flags' in operation: 211 lines.append(self.fmt.rst_fields('flags', 212 self.fmt.rst_list_inline(operation['flags']))) 213 214 if "do" in operation: 215 lines.append(self.fmt.rst_paragraph(":do:", 0)) 216 lines.append(self.parse_do(operation["do"], 0)) 217 if "dump" in operation: 218 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 219 lines.append(self.parse_do(operation["dump"], 0)) 220 if "event" in operation: 221 lines.append(self.fmt.rst_paragraph(":event:", 0)) 222 lines.append(self.parse_op_attributes(operation["event"], 0)) 223 224 # New line after fields 225 lines.append("\n") 226 227 return "\n".join(lines) 228 229 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 230 """Parse a list of entries""" 231 ignored = ["pad"] 232 lines = [] 233 for entry in entries: 234 if isinstance(entry, dict): 235 # entries could be a list or a dictionary 236 field_name = entry.get("name", "") 237 if field_name in ignored: 238 continue 239 type_ = entry.get("type") 240 if type_: 241 field_name += f" ({self.fmt.inline(type_)})" 242 lines.append( 243 self.fmt.rst_fields(field_name, 244 self.fmt.sanitize(entry.get("doc", "")), 245 level) 246 ) 247 elif isinstance(entry, list): 248 lines.append(self.fmt.rst_list_inline(entry, level)) 249 else: 250 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 251 level)) 252 253 lines.append("\n") 254 return "\n".join(lines) 255 256 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 257 """Parse definitions section""" 258 preprocessed = ["name", "entries", "members"] 259 ignored = ["render-max"] # This is not printed 260 lines = [] 261 262 for definition in defs: 263 if LINE_STR in definition: 264 lines.append(self.fmt.rst_lineno(definition[LINE_STR])) 265 266 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 267 for k in definition.keys(): 268 if k == LINE_STR: 269 continue 270 if k in preprocessed + ignored: 271 continue 272 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 273 274 # Field list needs to finish with a new line 275 lines.append("\n") 276 if "entries" in definition: 277 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 278 lines.append(self.parse_entries(definition["entries"], 1)) 279 if "members" in definition: 280 lines.append(self.fmt.rst_paragraph(":members:", 0)) 281 lines.append(self.parse_entries(definition["members"], 1)) 282 283 return "\n".join(lines) 284 285 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 286 """Parse attribute from attribute-set""" 287 preprocessed = ["name", "type"] 288 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 289 ignored = ["checks"] 290 lines = [] 291 292 for entry in entries: 293 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 294 entry["name"])) 295 296 if "doc" in entry: 297 lines.append(self.fmt.rst_paragraph(entry["doc"], 0) + "\n") 298 299 for attr in entry["attributes"]: 300 if LINE_STR in attr: 301 lines.append(self.fmt.rst_lineno(attr[LINE_STR])) 302 303 type_ = attr.get("type") 304 attr_line = attr["name"] 305 if type_: 306 # Add the attribute type in the same line 307 attr_line += f" ({self.fmt.inline(type_)})" 308 309 lines.append(self.fmt.rst_subsubsection(attr_line)) 310 311 for k in attr.keys(): 312 if k == LINE_STR: 313 continue 314 if k in preprocessed + ignored: 315 continue 316 if k in linkable: 317 value = self.fmt.rst_ref(namespace, k, attr[k]) 318 else: 319 value = self.fmt.sanitize(attr[k]) 320 lines.append(self.fmt.rst_fields(k, value, 0)) 321 lines.append("\n") 322 323 return "\n".join(lines) 324 325 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 326 """Parse sub-message definitions""" 327 lines = [] 328 329 for entry in entries: 330 lines.append(self.fmt.rst_section(namespace, 'sub-message', 331 entry["name"])) 332 for fmt in entry["formats"]: 333 value = fmt["value"] 334 335 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 336 for attr in ['fixed-header', 'attribute-set']: 337 if attr in fmt: 338 lines.append(self.fmt.rst_fields(attr, 339 self.fmt.rst_ref(namespace, 340 attr, 341 fmt[attr]), 342 1)) 343 lines.append("\n") 344 345 return "\n".join(lines) 346 347 def parse_yaml(self, obj: Dict[str, Any]) -> str: 348 """Format the whole YAML into a RST string""" 349 lines = [] 350 351 # Main header 352 lineno = obj.get('__lineno__', 0) 353 lines.append(self.fmt.rst_lineno(lineno)) 354 355 family = obj['name'] 356 357 lines.append(self.fmt.rst_header()) 358 lines.append(self.fmt.rst_label("netlink-" + family)) 359 360 title = f"Family ``{family}`` netlink specification" 361 lines.append(self.fmt.rst_title(title)) 362 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 363 364 if "doc" in obj: 365 lines.append(self.fmt.rst_subtitle("Summary")) 366 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 367 368 # Operations 369 if "operations" in obj: 370 lines.append(self.fmt.rst_subtitle("Operations")) 371 lines.append(self.parse_operations(obj["operations"]["list"], 372 family)) 373 374 # Multicast groups 375 if "mcast-groups" in obj: 376 lines.append(self.fmt.rst_subtitle("Multicast groups")) 377 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 378 379 # Definitions 380 if "definitions" in obj: 381 lines.append(self.fmt.rst_subtitle("Definitions")) 382 lines.append(self.parse_definitions(obj["definitions"], family)) 383 384 # Attributes set 385 if "attribute-sets" in obj: 386 lines.append(self.fmt.rst_subtitle("Attribute sets")) 387 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 388 389 # Sub-messages 390 if "sub-messages" in obj: 391 lines.append(self.fmt.rst_subtitle("Sub-messages")) 392 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 393 394 return "\n".join(lines) 395 396 # Main functions 397 # ============== 398 399 def parse_yaml_file(self, filename: str) -> str: 400 """Transform the YAML specified by filename into an RST-formatted string""" 401 with open(filename, "r", encoding="utf-8") as spec_file: 402 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 403 content = self.parse_yaml(numbered_yaml) 404 405 return content 406