1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 if prefix in mappings: 113 prefix = mappings[prefix] 114 return f":ref:`{namespace}-{prefix}-{name}`" 115 116 def rst_header(self) -> str: 117 """The headers for all the auto generated RST files""" 118 lines = [] 119 120 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 121 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 122 123 return "\n".join(lines) 124 125 @staticmethod 126 def rst_toctree(maxdepth: int = 2) -> str: 127 """Generate a toctree RST primitive""" 128 lines = [] 129 130 lines.append(".. toctree::") 131 lines.append(f" :maxdepth: {maxdepth}\n\n") 132 133 return "\n".join(lines) 134 135 @staticmethod 136 def rst_label(title: str) -> str: 137 """Return a formatted label""" 138 return f".. _{title}:\n\n" 139 140 @staticmethod 141 def rst_lineno(lineno: int) -> str: 142 """Return a lineno comment""" 143 return f".. LINENO {lineno}\n" 144 145class YnlDocGenerator: 146 """YAML Netlink specs Parser""" 147 148 fmt = RstFormatters() 149 150 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 151 """Parse 'multicast' group list and return a formatted string""" 152 lines = [] 153 for group in mcast_group: 154 lines.append(self.fmt.rst_bullet(group["name"])) 155 156 return "\n".join(lines) 157 158 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 159 """Parse 'do' section and return a formatted string""" 160 lines = [] 161 for key in do_dict.keys(): 162 if key == LINE_STR: 163 lines.append(self.fmt.rst_lineno(do_dict[key])) 164 continue 165 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 166 if key in ['request', 'reply']: 167 lines.append(self.parse_do_attributes(do_dict[key], level + 1) + "\n") 168 else: 169 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 170 171 return "\n".join(lines) 172 173 def parse_do_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 174 """Parse 'attributes' section""" 175 if "attributes" not in attrs: 176 return "" 177 lines = [self.fmt.rst_fields("attributes", 178 self.fmt.rst_list_inline(attrs["attributes"]), 179 level + 1)] 180 181 return "\n".join(lines) 182 183 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 184 """Parse operations block""" 185 preprocessed = ["name", "doc", "title", "do", "dump", "flags"] 186 linkable = ["fixed-header", "attribute-set"] 187 lines = [] 188 189 for operation in operations: 190 lines.append(self.fmt.rst_section(namespace, 'operation', 191 operation["name"])) 192 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 193 194 for key in operation.keys(): 195 if key == LINE_STR: 196 lines.append(self.fmt.rst_lineno(operation[key])) 197 continue 198 199 if key in preprocessed: 200 # Skip the special fields 201 continue 202 value = operation[key] 203 if key in linkable: 204 value = self.fmt.rst_ref(namespace, key, value) 205 lines.append(self.fmt.rst_fields(key, value, 0)) 206 if 'flags' in operation: 207 lines.append(self.fmt.rst_fields('flags', 208 self.fmt.rst_list_inline(operation['flags']))) 209 210 if "do" in operation: 211 lines.append(self.fmt.rst_paragraph(":do:", 0)) 212 lines.append(self.parse_do(operation["do"], 0)) 213 if "dump" in operation: 214 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 215 lines.append(self.parse_do(operation["dump"], 0)) 216 217 # New line after fields 218 lines.append("\n") 219 220 return "\n".join(lines) 221 222 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 223 """Parse a list of entries""" 224 ignored = ["pad"] 225 lines = [] 226 for entry in entries: 227 if isinstance(entry, dict): 228 # entries could be a list or a dictionary 229 field_name = entry.get("name", "") 230 if field_name in ignored: 231 continue 232 type_ = entry.get("type") 233 if type_: 234 field_name += f" ({self.fmt.inline(type_)})" 235 lines.append( 236 self.fmt.rst_fields(field_name, 237 self.fmt.sanitize(entry.get("doc", "")), 238 level) 239 ) 240 elif isinstance(entry, list): 241 lines.append(self.fmt.rst_list_inline(entry, level)) 242 else: 243 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 244 level)) 245 246 lines.append("\n") 247 return "\n".join(lines) 248 249 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 250 """Parse definitions section""" 251 preprocessed = ["name", "entries", "members"] 252 ignored = ["render-max"] # This is not printed 253 lines = [] 254 255 for definition in defs: 256 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 257 for k in definition.keys(): 258 if k == LINE_STR: 259 lines.append(self.fmt.rst_lineno(definition[k])) 260 continue 261 if k in preprocessed + ignored: 262 continue 263 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 264 265 # Field list needs to finish with a new line 266 lines.append("\n") 267 if "entries" in definition: 268 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 269 lines.append(self.parse_entries(definition["entries"], 1)) 270 if "members" in definition: 271 lines.append(self.fmt.rst_paragraph(":members:", 0)) 272 lines.append(self.parse_entries(definition["members"], 1)) 273 274 return "\n".join(lines) 275 276 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 277 """Parse attribute from attribute-set""" 278 preprocessed = ["name", "type"] 279 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 280 ignored = ["checks"] 281 lines = [] 282 283 for entry in entries: 284 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 285 entry["name"])) 286 for attr in entry["attributes"]: 287 type_ = attr.get("type") 288 attr_line = attr["name"] 289 if type_: 290 # Add the attribute type in the same line 291 attr_line += f" ({self.fmt.inline(type_)})" 292 293 lines.append(self.fmt.rst_subsubsection(attr_line)) 294 295 for k in attr.keys(): 296 if k == LINE_STR: 297 lines.append(self.fmt.rst_lineno(attr[k])) 298 continue 299 if k in preprocessed + ignored: 300 continue 301 if k in linkable: 302 value = self.fmt.rst_ref(namespace, k, attr[k]) 303 else: 304 value = self.fmt.sanitize(attr[k]) 305 lines.append(self.fmt.rst_fields(k, value, 0)) 306 lines.append("\n") 307 308 return "\n".join(lines) 309 310 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 311 """Parse sub-message definitions""" 312 lines = [] 313 314 for entry in entries: 315 lines.append(self.fmt.rst_section(namespace, 'sub-message', 316 entry["name"])) 317 for fmt in entry["formats"]: 318 value = fmt["value"] 319 320 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 321 for attr in ['fixed-header', 'attribute-set']: 322 if attr in fmt: 323 lines.append(self.fmt.rst_fields(attr, 324 self.fmt.rst_ref(namespace, 325 attr, 326 fmt[attr]), 327 1)) 328 lines.append("\n") 329 330 return "\n".join(lines) 331 332 def parse_yaml(self, obj: Dict[str, Any]) -> str: 333 """Format the whole YAML into a RST string""" 334 lines = [] 335 336 # Main header 337 lineno = obj.get('__lineno__', 0) 338 lines.append(self.fmt.rst_lineno(lineno)) 339 340 family = obj['name'] 341 342 lines.append(self.fmt.rst_header()) 343 lines.append(self.fmt.rst_label("netlink-" + family)) 344 345 title = f"Family ``{family}`` netlink specification" 346 lines.append(self.fmt.rst_title(title)) 347 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 348 349 if "doc" in obj: 350 lines.append(self.fmt.rst_subtitle("Summary")) 351 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 352 353 # Operations 354 if "operations" in obj: 355 lines.append(self.fmt.rst_subtitle("Operations")) 356 lines.append(self.parse_operations(obj["operations"]["list"], 357 family)) 358 359 # Multicast groups 360 if "mcast-groups" in obj: 361 lines.append(self.fmt.rst_subtitle("Multicast groups")) 362 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 363 364 # Definitions 365 if "definitions" in obj: 366 lines.append(self.fmt.rst_subtitle("Definitions")) 367 lines.append(self.parse_definitions(obj["definitions"], family)) 368 369 # Attributes set 370 if "attribute-sets" in obj: 371 lines.append(self.fmt.rst_subtitle("Attribute sets")) 372 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 373 374 # Sub-messages 375 if "sub-messages" in obj: 376 lines.append(self.fmt.rst_subtitle("Sub-messages")) 377 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 378 379 return "\n".join(lines) 380 381 # Main functions 382 # ============== 383 384 def parse_yaml_file(self, filename: str) -> str: 385 """Transform the YAML specified by filename into an RST-formatted string""" 386 with open(filename, "r", encoding="utf-8") as spec_file: 387 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 388 content = self.parse_yaml(numbered_yaml) 389 390 return content 391