1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# -*- coding: utf-8; mode: python -*- 4 5""" 6 Class to auto generate the documentation for Netlink specifications. 7 8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org> 9 :license: GPL Version 2, June 1991 see linux/COPYING for details. 10 11 This class performs extensive parsing to the Linux kernel's netlink YAML 12 spec files, in an effort to avoid needing to heavily mark up the original 13 YAML file. 14 15 This code is split in two classes: 16 1) RST formatters: Use to convert a string to a RST output 17 2) YAML Netlink (YNL) doc generator: Generate docs from YAML data 18""" 19 20from typing import Any, Dict, List 21import yaml 22 23LINE_STR = '__lineno__' 24 25class NumberedSafeLoader(yaml.SafeLoader): # pylint: disable=R0901 26 """Override the SafeLoader class to add line number to parsed data""" 27 28 def construct_mapping(self, node, *args, **kwargs): 29 mapping = super().construct_mapping(node, *args, **kwargs) 30 mapping[LINE_STR] = node.start_mark.line 31 32 return mapping 33 34class RstFormatters: 35 """RST Formatters""" 36 37 SPACE_PER_LEVEL = 4 38 39 @staticmethod 40 def headroom(level: int) -> str: 41 """Return space to format""" 42 return " " * (level * RstFormatters.SPACE_PER_LEVEL) 43 44 @staticmethod 45 def bold(text: str) -> str: 46 """Format bold text""" 47 return f"**{text}**" 48 49 @staticmethod 50 def inline(text: str) -> str: 51 """Format inline text""" 52 return f"``{text}``" 53 54 @staticmethod 55 def sanitize(text: str) -> str: 56 """Remove newlines and multiple spaces""" 57 # This is useful for some fields that are spread across multiple lines 58 return str(text).replace("\n", " ").strip() 59 60 def rst_fields(self, key: str, value: str, level: int = 0) -> str: 61 """Return a RST formatted field""" 62 return self.headroom(level) + f":{key}: {value}" 63 64 def rst_definition(self, key: str, value: Any, level: int = 0) -> str: 65 """Format a single rst definition""" 66 return self.headroom(level) + key + "\n" + self.headroom(level + 1) + str(value) 67 68 def rst_paragraph(self, paragraph: str, level: int = 0) -> str: 69 """Return a formatted paragraph""" 70 return self.headroom(level) + paragraph 71 72 def rst_bullet(self, item: str, level: int = 0) -> str: 73 """Return a formatted a bullet""" 74 return self.headroom(level) + f"- {item}" 75 76 @staticmethod 77 def rst_subsection(title: str) -> str: 78 """Add a sub-section to the document""" 79 return f"{title}\n" + "-" * len(title) 80 81 @staticmethod 82 def rst_subsubsection(title: str) -> str: 83 """Add a sub-sub-section to the document""" 84 return f"{title}\n" + "~" * len(title) 85 86 @staticmethod 87 def rst_section(namespace: str, prefix: str, title: str) -> str: 88 """Add a section to the document""" 89 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title) 90 91 @staticmethod 92 def rst_subtitle(title: str) -> str: 93 """Add a subtitle to the document""" 94 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n" 95 96 @staticmethod 97 def rst_title(title: str) -> str: 98 """Add a title to the document""" 99 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n" 100 101 def rst_list_inline(self, list_: List[str], level: int = 0) -> str: 102 """Format a list using inlines""" 103 return self.headroom(level) + "[" + ", ".join(self.inline(i) for i in list_) + "]" 104 105 @staticmethod 106 def rst_ref(namespace: str, prefix: str, name: str) -> str: 107 """Add a hyperlink to the document""" 108 mappings = {'enum': 'definition', 109 'fixed-header': 'definition', 110 'nested-attributes': 'attribute-set', 111 'struct': 'definition'} 112 prefix = mappings.get(prefix, prefix) 113 return f":ref:`{namespace}-{prefix}-{name}`" 114 115 def rst_header(self) -> str: 116 """The headers for all the auto generated RST files""" 117 lines = [] 118 119 lines.append(self.rst_paragraph(".. SPDX-License-Identifier: GPL-2.0")) 120 lines.append(self.rst_paragraph(".. NOTE: This document was auto-generated.\n\n")) 121 122 return "\n".join(lines) 123 124 @staticmethod 125 def rst_toctree(maxdepth: int = 2) -> str: 126 """Generate a toctree RST primitive""" 127 lines = [] 128 129 lines.append(".. toctree::") 130 lines.append(f" :maxdepth: {maxdepth}\n\n") 131 132 return "\n".join(lines) 133 134 @staticmethod 135 def rst_label(title: str) -> str: 136 """Return a formatted label""" 137 return f".. _{title}:\n\n" 138 139 @staticmethod 140 def rst_lineno(lineno: int) -> str: 141 """Return a lineno comment""" 142 return f".. LINENO {lineno}\n" 143 144class YnlDocGenerator: 145 """YAML Netlink specs Parser""" 146 147 fmt = RstFormatters() 148 149 def parse_mcast_group(self, mcast_group: List[Dict[str, Any]]) -> str: 150 """Parse 'multicast' group list and return a formatted string""" 151 lines = [] 152 for group in mcast_group: 153 lines.append(self.fmt.rst_bullet(group["name"])) 154 155 return "\n".join(lines) 156 157 def parse_do(self, do_dict: Dict[str, Any], level: int = 0) -> str: 158 """Parse 'do' section and return a formatted string""" 159 lines = [] 160 if LINE_STR in do_dict: 161 lines.append(self.fmt.rst_lineno(do_dict[LINE_STR])) 162 163 for key in do_dict.keys(): 164 if key == LINE_STR: 165 continue 166 lines.append(self.fmt.rst_paragraph(self.fmt.bold(key), level + 1)) 167 if key in ['request', 'reply']: 168 lines.append(self.parse_op_attributes(do_dict[key], level + 1) + "\n") 169 else: 170 lines.append(self.fmt.headroom(level + 2) + do_dict[key] + "\n") 171 172 return "\n".join(lines) 173 174 def parse_op_attributes(self, attrs: Dict[str, Any], level: int = 0) -> str: 175 """Parse 'attributes' section""" 176 if "attributes" not in attrs: 177 return "" 178 lines = [self.fmt.rst_fields("attributes", 179 self.fmt.rst_list_inline(attrs["attributes"]), 180 level + 1)] 181 182 return "\n".join(lines) 183 184 def parse_operations(self, operations: List[Dict[str, Any]], namespace: str) -> str: 185 """Parse operations block""" 186 preprocessed = ["name", "doc", "title", "do", "dump", "flags", "event"] 187 linkable = ["fixed-header", "attribute-set"] 188 lines = [] 189 190 for operation in operations: 191 if LINE_STR in operation: 192 lines.append(self.fmt.rst_lineno(operation[LINE_STR])) 193 194 lines.append(self.fmt.rst_section(namespace, 'operation', 195 operation["name"])) 196 lines.append(self.fmt.rst_paragraph(operation["doc"]) + "\n") 197 198 for key in operation.keys(): 199 if key == LINE_STR: 200 continue 201 202 if key in preprocessed: 203 # Skip the special fields 204 continue 205 value = operation[key] 206 if key in linkable: 207 value = self.fmt.rst_ref(namespace, key, value) 208 lines.append(self.fmt.rst_fields(key, value, 0)) 209 if 'flags' in operation: 210 lines.append(self.fmt.rst_fields('flags', 211 self.fmt.rst_list_inline(operation['flags']))) 212 213 if "do" in operation: 214 lines.append(self.fmt.rst_paragraph(":do:", 0)) 215 lines.append(self.parse_do(operation["do"], 0)) 216 if "dump" in operation: 217 lines.append(self.fmt.rst_paragraph(":dump:", 0)) 218 lines.append(self.parse_do(operation["dump"], 0)) 219 if "event" in operation: 220 lines.append(self.fmt.rst_paragraph(":event:", 0)) 221 lines.append(self.parse_op_attributes(operation["event"], 0)) 222 223 # New line after fields 224 lines.append("\n") 225 226 return "\n".join(lines) 227 228 def parse_entries(self, entries: List[Dict[str, Any]], level: int) -> str: 229 """Parse a list of entries""" 230 ignored = ["pad"] 231 lines = [] 232 for entry in entries: 233 if isinstance(entry, dict): 234 # entries could be a list or a dictionary 235 field_name = entry.get("name", "") 236 if field_name in ignored: 237 continue 238 type_ = entry.get("type") 239 if type_: 240 field_name += f" ({self.fmt.inline(type_)})" 241 lines.append( 242 self.fmt.rst_fields(field_name, 243 self.fmt.sanitize(entry.get("doc", "")), 244 level) 245 ) 246 elif isinstance(entry, list): 247 lines.append(self.fmt.rst_list_inline(entry, level)) 248 else: 249 lines.append(self.fmt.rst_bullet(self.fmt.inline(self.fmt.sanitize(entry)), 250 level)) 251 252 lines.append("\n") 253 return "\n".join(lines) 254 255 def parse_definitions(self, defs: Dict[str, Any], namespace: str) -> str: 256 """Parse definitions section""" 257 preprocessed = ["name", "entries", "members"] 258 ignored = ["render-max"] # This is not printed 259 lines = [] 260 261 for definition in defs: 262 if LINE_STR in definition: 263 lines.append(self.fmt.rst_lineno(definition[LINE_STR])) 264 265 lines.append(self.fmt.rst_section(namespace, 'definition', definition["name"])) 266 for k in definition.keys(): 267 if k == LINE_STR: 268 continue 269 if k in preprocessed + ignored: 270 continue 271 lines.append(self.fmt.rst_fields(k, self.fmt.sanitize(definition[k]), 0)) 272 273 # Field list needs to finish with a new line 274 lines.append("\n") 275 if "entries" in definition: 276 lines.append(self.fmt.rst_paragraph(":entries:", 0)) 277 lines.append(self.parse_entries(definition["entries"], 1)) 278 if "members" in definition: 279 lines.append(self.fmt.rst_paragraph(":members:", 0)) 280 lines.append(self.parse_entries(definition["members"], 1)) 281 282 return "\n".join(lines) 283 284 def parse_attr_sets(self, entries: List[Dict[str, Any]], namespace: str) -> str: 285 """Parse attribute from attribute-set""" 286 preprocessed = ["name", "type"] 287 linkable = ["enum", "nested-attributes", "struct", "sub-message"] 288 ignored = ["checks"] 289 lines = [] 290 291 for entry in entries: 292 lines.append(self.fmt.rst_section(namespace, 'attribute-set', 293 entry["name"])) 294 295 if "doc" in entry: 296 lines.append(self.fmt.rst_paragraph(entry["doc"], 0) + "\n") 297 298 for attr in entry["attributes"]: 299 if LINE_STR in attr: 300 lines.append(self.fmt.rst_lineno(attr[LINE_STR])) 301 302 type_ = attr.get("type") 303 attr_line = attr["name"] 304 if type_: 305 # Add the attribute type in the same line 306 attr_line += f" ({self.fmt.inline(type_)})" 307 308 lines.append(self.fmt.rst_subsubsection(attr_line)) 309 310 for k in attr.keys(): 311 if k == LINE_STR: 312 continue 313 if k in preprocessed + ignored: 314 continue 315 if k in linkable: 316 value = self.fmt.rst_ref(namespace, k, attr[k]) 317 else: 318 value = self.fmt.sanitize(attr[k]) 319 lines.append(self.fmt.rst_fields(k, value, 0)) 320 lines.append("\n") 321 322 return "\n".join(lines) 323 324 def parse_sub_messages(self, entries: List[Dict[str, Any]], namespace: str) -> str: 325 """Parse sub-message definitions""" 326 lines = [] 327 328 for entry in entries: 329 lines.append(self.fmt.rst_section(namespace, 'sub-message', 330 entry["name"])) 331 for fmt in entry["formats"]: 332 value = fmt["value"] 333 334 lines.append(self.fmt.rst_bullet(self.fmt.bold(value))) 335 for attr in ['fixed-header', 'attribute-set']: 336 if attr in fmt: 337 lines.append(self.fmt.rst_fields(attr, 338 self.fmt.rst_ref(namespace, 339 attr, 340 fmt[attr]), 341 1)) 342 lines.append("\n") 343 344 return "\n".join(lines) 345 346 def parse_yaml(self, obj: Dict[str, Any]) -> str: 347 """Format the whole YAML into a RST string""" 348 lines = [] 349 350 # Main header 351 lineno = obj.get('__lineno__', 0) 352 lines.append(self.fmt.rst_lineno(lineno)) 353 354 family = obj['name'] 355 356 lines.append(self.fmt.rst_header()) 357 lines.append(self.fmt.rst_label("netlink-" + family)) 358 359 title = f"Family ``{family}`` netlink specification" 360 lines.append(self.fmt.rst_title(title)) 361 lines.append(self.fmt.rst_paragraph(".. contents:: :depth: 3\n")) 362 363 if "doc" in obj: 364 lines.append(self.fmt.rst_subtitle("Summary")) 365 lines.append(self.fmt.rst_paragraph(obj["doc"], 0)) 366 367 # Operations 368 if "operations" in obj: 369 lines.append(self.fmt.rst_subtitle("Operations")) 370 lines.append(self.parse_operations(obj["operations"]["list"], 371 family)) 372 373 # Multicast groups 374 if "mcast-groups" in obj: 375 lines.append(self.fmt.rst_subtitle("Multicast groups")) 376 lines.append(self.parse_mcast_group(obj["mcast-groups"]["list"])) 377 378 # Definitions 379 if "definitions" in obj: 380 lines.append(self.fmt.rst_subtitle("Definitions")) 381 lines.append(self.parse_definitions(obj["definitions"], family)) 382 383 # Attributes set 384 if "attribute-sets" in obj: 385 lines.append(self.fmt.rst_subtitle("Attribute sets")) 386 lines.append(self.parse_attr_sets(obj["attribute-sets"], family)) 387 388 # Sub-messages 389 if "sub-messages" in obj: 390 lines.append(self.fmt.rst_subtitle("Sub-messages")) 391 lines.append(self.parse_sub_messages(obj["sub-messages"], family)) 392 393 return "\n".join(lines) 394 395 # Main functions 396 # ============== 397 398 def parse_yaml_file(self, filename: str) -> str: 399 """Transform the YAML specified by filename into an RST-formatted string""" 400 with open(filename, "r", encoding="utf-8") as spec_file: 401 numbered_yaml = yaml.load(spec_file, Loader=NumberedSafeLoader) 402 content = self.parse_yaml(numbered_yaml) 403 404 return content 405