1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import os 49import shlex 50import shutil 51import subprocess 52import sys 53 54from lib.python_version import PythonVersion 55from lib.latex_fonts import LatexFontChecker 56 57LIB_DIR = "../../scripts/lib" 58SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 59 60sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 61 62from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 63 64# 65# Some constants 66# 67MIN_PYTHON_VERSION = PythonVersion("3.7").version 68PAPER = ["", "a4", "letter"] 69 70TARGETS = { 71 "cleandocs": { "builder": "clean" }, 72 "linkcheckdocs": { "builder": "linkcheck" }, 73 "htmldocs": { "builder": "html" }, 74 "epubdocs": { "builder": "epub", "out_dir": "epub" }, 75 "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 76 "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 77 "latexdocs": { "builder": "latex", "out_dir": "latex" }, 78 "pdfdocs": { "builder": "latex", "out_dir": "latex" }, 79 "xmldocs": { "builder": "xml", "out_dir": "xml" }, 80} 81 82 83# 84# SphinxBuilder class 85# 86 87class SphinxBuilder: 88 """ 89 Handles a sphinx-build target, adding needed arguments to build 90 with the Kernel. 91 """ 92 93 def is_rust_enabled(self): 94 """Check if rust is enabled at .config""" 95 config_path = os.path.join(self.srctree, ".config") 96 if os.path.isfile(config_path): 97 with open(config_path, "r", encoding="utf-8") as f: 98 return "CONFIG_RUST=y" in f.read() 99 return False 100 101 def get_path(self, path, use_cwd=False, abs_path=False): 102 """ 103 Ancillary routine to handle patches the right way, as shell does. 104 105 It first expands "~" and "~user". Then, if patch is not absolute, 106 join self.srctree. Finally, if requested, convert to abspath. 107 """ 108 109 path = os.path.expanduser(path) 110 if not path.startswith("/"): 111 if use_cwd: 112 base = os.getcwd() 113 else: 114 base = self.srctree 115 116 path = os.path.join(base, path) 117 118 if abs_path: 119 return os.path.abspath(path) 120 121 return path 122 123 def get_sphinx_extra_opts(self, n_jobs): 124 """ 125 Get the number of jobs to be used for docs build passed via command 126 line and desired sphinx verbosity. 127 128 The number of jobs can be on different places: 129 130 1) It can be passed via "-j" argument; 131 2) The SPHINXOPTS="-j8" env var may have "-j"; 132 3) if called via GNU make, -j specifies the desired number of jobs. 133 with GNU makefile, this number is available via POSIX jobserver; 134 4) if none of the above is available, it should default to "-jauto", 135 and let sphinx decide the best value. 136 """ 137 138 # 139 # SPHINXOPTS env var, if used, contains extra arguments to be used 140 # by sphinx-build time. Among them, it may contain sphinx verbosity 141 # and desired number of parallel jobs. 142 # 143 parser = argparse.ArgumentParser() 144 parser.add_argument('-j', '--jobs', type=int) 145 parser.add_argument('-q', '--quiet', type=int) 146 147 # 148 # Other sphinx-build arguments go as-is, so place them 149 # at self.sphinxopts, using shell parser 150 # 151 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 152 153 # 154 # Build a list of sphinx args, honoring verbosity here if specified 155 # 156 157 verbose = self.verbose 158 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 159 if sphinx_args.quiet is True: 160 verbose = False 161 162 # 163 # If the user explicitly sets "-j" at command line, use it. 164 # Otherwise, pick it from SPHINXOPTS args 165 # 166 if n_jobs: 167 self.n_jobs = n_jobs 168 elif sphinx_args.jobs: 169 self.n_jobs = sphinx_args.jobs 170 else: 171 self.n_jobs = None 172 173 if not verbose: 174 self.sphinxopts += ["-q"] 175 176 def __init__(self, builddir, verbose=False, n_jobs=None): 177 """Initialize internal variables""" 178 self.verbose = None 179 180 # 181 # Normal variables passed from Kernel's makefile 182 # 183 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 184 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 185 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 186 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 187 188 if not verbose: 189 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 190 191 if verbose is not None: 192 self.verbose = verbose 193 194 # 195 # Source tree directory. This needs to be at os.environ, as 196 # Sphinx extensions use it 197 # 198 self.srctree = os.environ.get("srctree") 199 if not self.srctree: 200 self.srctree = "." 201 os.environ["srctree"] = self.srctree 202 203 # 204 # Now that we can expand srctree, get other directories as well 205 # 206 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 207 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 208 "scripts/kernel-doc.py")) 209 self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) 210 211 self.config_rust = self.is_rust_enabled() 212 213 # 214 # Get directory locations for LaTeX build toolchain 215 # 216 self.pdflatex_cmd = shutil.which(self.pdflatex) 217 self.latexmk_cmd = shutil.which("latexmk") 218 219 self.env = os.environ.copy() 220 221 self.get_sphinx_extra_opts(n_jobs) 222 223 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 224 """ 225 Executes sphinx-build using current python3 command. 226 227 When calling via GNU make, POSIX jobserver is used to tell how 228 many jobs are still available from a job pool. claim all remaining 229 jobs, as we don't want sphinx-build to run in parallel with other 230 jobs. 231 232 Despite that, the user may actually force a different value than 233 the number of available jobs via command line. 234 235 The "with" logic here is used to ensure that the claimed jobs will 236 be freed once subprocess finishes 237 """ 238 239 with JobserverExec() as jobserver: 240 if jobserver.claim: 241 # 242 # when GNU make is used, claim available jobs from jobserver 243 # 244 n_jobs = str(jobserver.claim) 245 else: 246 # 247 # Otherwise, let sphinx decide by default 248 # 249 n_jobs = "auto" 250 251 # 252 # If explicitly requested via command line, override default 253 # 254 if self.n_jobs: 255 n_jobs = str(self.n_jobs) 256 257 cmd = [sys.executable, sphinx_build] 258 cmd += [f"-j{n_jobs}"] 259 cmd += self.sphinxopts 260 cmd += build_args 261 262 if self.verbose: 263 print(" ".join(cmd)) 264 265 return subprocess.call(cmd, *args, **pwargs) 266 267 def handle_html(self, css, output_dir): 268 """ 269 Extra steps for HTML and epub output. 270 271 For such targets, we need to ensure that CSS will be properly 272 copied to the output _static directory 273 """ 274 275 if not css: 276 return 277 278 css = os.path.expanduser(css) 279 if not css.startswith("/"): 280 css = os.path.join(self.srctree, css) 281 282 static_dir = os.path.join(output_dir, "_static") 283 os.makedirs(static_dir, exist_ok=True) 284 285 try: 286 shutil.copy2(css, static_dir) 287 except (OSError, IOError) as e: 288 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 289 290 def handle_pdf(self, output_dirs, deny_vf): 291 """ 292 Extra steps for PDF output. 293 294 As PDF is handled via a LaTeX output, after building the .tex file, 295 a new build is needed to create the PDF output from the latex 296 directory. 297 """ 298 builds = {} 299 max_len = 0 300 301 # 302 # Since early 2024, Fedora and openSUSE tumbleweed have started 303 # deploying variable-font format of "Noto CJK", causing LaTeX 304 # to break with CJK. Work around it, by denying the variable font 305 # usage during xelatex build by passing the location of a config 306 # file with a deny list. 307 # 308 # See tools/docs/lib/latex_fonts.py for more details. 309 # 310 if deny_vf: 311 deny_vf = os.path.expanduser(deny_vf) 312 if os.path.isdir(deny_vf): 313 self.env["XDG_CONFIG_HOME"] = deny_vf 314 315 for from_dir in output_dirs: 316 pdf_dir = os.path.join(from_dir, "../pdf") 317 os.makedirs(pdf_dir, exist_ok=True) 318 319 if self.latexmk_cmd: 320 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 321 else: 322 latex_cmd = [self.pdflatex] 323 324 latex_cmd.extend(shlex.split(self.latexopts)) 325 326 tex_suffix = ".tex" 327 328 # 329 # Process each .tex file 330 # 331 332 has_tex = False 333 build_failed = False 334 with os.scandir(from_dir) as it: 335 for entry in it: 336 if not entry.name.endswith(tex_suffix): 337 continue 338 339 name = entry.name[:-len(tex_suffix)] 340 has_tex = True 341 342 # 343 # LaTeX PDF error code is almost useless for us: 344 # any warning makes it non-zero. For kernel doc builds it 345 # always return non-zero even when build succeeds. 346 # So, let's do the best next thing: check if all PDF 347 # files were built. If they're, print a summary and 348 # return 0 at the end of this function 349 # 350 try: 351 subprocess.run(latex_cmd + [entry.path], 352 cwd=from_dir, check=True, env=self.env) 353 except subprocess.CalledProcessError: 354 pass 355 356 pdf_name = name + ".pdf" 357 pdf_from = os.path.join(from_dir, pdf_name) 358 pdf_to = os.path.join(pdf_dir, pdf_name) 359 360 if os.path.exists(pdf_from): 361 os.rename(pdf_from, pdf_to) 362 builds[name] = os.path.relpath(pdf_to, self.builddir) 363 else: 364 builds[name] = "FAILED" 365 build_failed = True 366 367 name = entry.name.removesuffix(".tex") 368 max_len = max(max_len, len(name)) 369 370 if not has_tex: 371 name = os.path.basename(from_dir) 372 max_len = max(max_len, len(name)) 373 builds[name] = "FAILED (no .tex)" 374 build_failed = True 375 376 msg = "Summary" 377 msg += "\n" + "=" * len(msg) 378 print() 379 print(msg) 380 381 for pdf_name, pdf_file in builds.items(): 382 print(f"{pdf_name:<{max_len}}: {pdf_file}") 383 384 print() 385 386 if build_failed: 387 msg = LatexFontChecker().check() 388 if msg: 389 print(msg) 390 391 sys.exit("PDF build failed: not all PDF files were created.") 392 else: 393 print("All PDF files were built.") 394 395 def handle_info(self, output_dirs): 396 """ 397 Extra steps for Info output. 398 399 For texinfo generation, an additional make is needed from the 400 texinfo directory. 401 """ 402 403 for output_dir in output_dirs: 404 try: 405 subprocess.run(["make", "info"], cwd=output_dir, check=True) 406 except subprocess.CalledProcessError as e: 407 sys.exit(f"Error generating info docs: {e}") 408 409 def cleandocs(self, builder): # pylint: disable=W0613 410 """Remove documentation output directory""" 411 shutil.rmtree(self.builddir, ignore_errors=True) 412 413 def build(self, target, sphinxdirs=None, conf="conf.py", 414 theme=None, css=None, paper=None, deny_vf=None): 415 """ 416 Build documentation using Sphinx. This is the core function of this 417 module. It prepares all arguments required by sphinx-build. 418 """ 419 420 builder = TARGETS[target]["builder"] 421 out_dir = TARGETS[target].get("out_dir", "") 422 423 # 424 # Cleandocs doesn't require sphinx-build 425 # 426 if target == "cleandocs": 427 self.cleandocs(builder) 428 return 429 430 if theme: 431 os.environ["DOCS_THEME"] = theme 432 433 # 434 # Other targets require sphinx-build, so check if it exists 435 # 436 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 437 if not sphinxbuild: 438 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 439 440 if builder == "latex": 441 if not self.pdflatex_cmd and not self.latexmk_cmd: 442 sys.exit("Error: pdflatex or latexmk required for PDF generation") 443 444 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 445 446 # 447 # Fill in base arguments for Sphinx build 448 # 449 kerneldoc = self.kerneldoc 450 if kerneldoc.startswith(self.srctree): 451 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 452 453 args = [ "-b", builder, "-c", docs_dir ] 454 455 if builder == "latex": 456 if not paper: 457 paper = PAPER[1] 458 459 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 460 461 if self.config_rust: 462 args.extend(["-t", "rustdoc"]) 463 464 if conf: 465 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 466 467 if not sphinxdirs: 468 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 469 470 # 471 # sphinxdirs can be a list or a whitespace-separated string 472 # 473 sphinxdirs_list = [] 474 for sphinxdir in sphinxdirs: 475 if isinstance(sphinxdir, list): 476 sphinxdirs_list += sphinxdir 477 else: 478 sphinxdirs_list += sphinxdir.split() 479 480 # 481 # Step 1: Build each directory in separate. 482 # 483 # This is not the best way of handling it, as cross-references between 484 # them will be broken, but this is what we've been doing since 485 # the beginning. 486 # 487 output_dirs = [] 488 for sphinxdir in sphinxdirs_list: 489 src_dir = os.path.join(docs_dir, sphinxdir) 490 doctree_dir = os.path.join(self.builddir, ".doctrees") 491 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 492 493 # 494 # Make directory names canonical 495 # 496 src_dir = os.path.normpath(src_dir) 497 doctree_dir = os.path.normpath(doctree_dir) 498 output_dir = os.path.normpath(output_dir) 499 500 os.makedirs(doctree_dir, exist_ok=True) 501 os.makedirs(output_dir, exist_ok=True) 502 503 output_dirs.append(output_dir) 504 505 build_args = args + [ 506 "-d", doctree_dir, 507 "-D", f"kerneldoc_bin={kerneldoc}", 508 "-D", f"version={self.kernelversion}", 509 "-D", f"release={self.kernelrelease}", 510 "-D", f"kerneldoc_srctree={self.srctree}", 511 src_dir, 512 output_dir, 513 ] 514 515 try: 516 self.run_sphinx(sphinxbuild, build_args, env=self.env) 517 except (OSError, ValueError, subprocess.SubprocessError) as e: 518 sys.exit(f"Build failed: {repr(e)}") 519 520 # 521 # Ensure that each html/epub output will have needed static files 522 # 523 if target in ["htmldocs", "epubdocs"]: 524 self.handle_html(css, output_dir) 525 526 # 527 # Step 2: Some targets (PDF and info) require an extra step once 528 # sphinx-build finishes 529 # 530 if target == "pdfdocs": 531 self.handle_pdf(output_dirs, deny_vf) 532 elif target == "infodocs": 533 self.handle_info(output_dirs) 534 535def jobs_type(value): 536 """ 537 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 538 equal or bigger than one. 539 """ 540 if value is None: 541 return None 542 543 if value.lower() == 'auto': 544 return value.lower() 545 546 try: 547 if int(value) >= 1: 548 return value 549 550 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 551 except ValueError: 552 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 553 554def main(): 555 """ 556 Main function. The only mandatory argument is the target. If not 557 specified, the other arguments will use default values if not 558 specified at os.environ. 559 """ 560 parser = argparse.ArgumentParser(description="Kernel documentation builder") 561 562 parser.add_argument("target", choices=list(TARGETS.keys()), 563 help="Documentation target to build") 564 parser.add_argument("--sphinxdirs", nargs="+", 565 help="Specific directories to build") 566 parser.add_argument("--conf", default="conf.py", 567 help="Sphinx configuration file") 568 parser.add_argument("--builddir", default="output", 569 help="Sphinx configuration file") 570 571 parser.add_argument("--theme", help="Sphinx theme to use") 572 573 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 574 575 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 576 help="Paper size for LaTeX/PDF output") 577 578 parser.add_argument('--deny-vf', 579 help="Configuration to deny variable fonts on pdf builds") 580 581 parser.add_argument("-v", "--verbose", action='store_true', 582 help="place build in verbose mode") 583 584 parser.add_argument('-j', '--jobs', type=jobs_type, 585 help="Sets number of jobs to use with sphinx-build") 586 587 args = parser.parse_args() 588 589 PythonVersion.check_python(MIN_PYTHON_VERSION) 590 591 builder = SphinxBuilder(builddir=args.builddir, 592 verbose=args.verbose, n_jobs=args.jobs) 593 594 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 595 theme=args.theme, css=args.css, paper=args.paper, 596 deny_vf=args.deny_vf) 597 598if __name__ == "__main__": 599 main() 600