1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import os 49import shlex 50import shutil 51import subprocess 52import sys 53 54from concurrent import futures 55 56from lib.python_version import PythonVersion 57from lib.latex_fonts import LatexFontChecker 58 59LIB_DIR = "../../scripts/lib" 60SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 61 62sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 63 64from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 65 66# 67# Some constants 68# 69MIN_PYTHON_VERSION = PythonVersion("3.7").version 70PAPER = ["", "a4", "letter"] 71 72TARGETS = { 73 "cleandocs": { "builder": "clean" }, 74 "linkcheckdocs": { "builder": "linkcheck" }, 75 "htmldocs": { "builder": "html" }, 76 "epubdocs": { "builder": "epub", "out_dir": "epub" }, 77 "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 78 "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 79 "latexdocs": { "builder": "latex", "out_dir": "latex" }, 80 "pdfdocs": { "builder": "latex", "out_dir": "latex" }, 81 "xmldocs": { "builder": "xml", "out_dir": "xml" }, 82} 83 84 85# 86# SphinxBuilder class 87# 88 89class SphinxBuilder: 90 """ 91 Handles a sphinx-build target, adding needed arguments to build 92 with the Kernel. 93 """ 94 95 def is_rust_enabled(self): 96 """Check if rust is enabled at .config""" 97 config_path = os.path.join(self.srctree, ".config") 98 if os.path.isfile(config_path): 99 with open(config_path, "r", encoding="utf-8") as f: 100 return "CONFIG_RUST=y" in f.read() 101 return False 102 103 def get_path(self, path, use_cwd=False, abs_path=False): 104 """ 105 Ancillary routine to handle patches the right way, as shell does. 106 107 It first expands "~" and "~user". Then, if patch is not absolute, 108 join self.srctree. Finally, if requested, convert to abspath. 109 """ 110 111 path = os.path.expanduser(path) 112 if not path.startswith("/"): 113 if use_cwd: 114 base = os.getcwd() 115 else: 116 base = self.srctree 117 118 path = os.path.join(base, path) 119 120 if abs_path: 121 return os.path.abspath(path) 122 123 return path 124 125 def get_sphinx_extra_opts(self, n_jobs): 126 """ 127 Get the number of jobs to be used for docs build passed via command 128 line and desired sphinx verbosity. 129 130 The number of jobs can be on different places: 131 132 1) It can be passed via "-j" argument; 133 2) The SPHINXOPTS="-j8" env var may have "-j"; 134 3) if called via GNU make, -j specifies the desired number of jobs. 135 with GNU makefile, this number is available via POSIX jobserver; 136 4) if none of the above is available, it should default to "-jauto", 137 and let sphinx decide the best value. 138 """ 139 140 # 141 # SPHINXOPTS env var, if used, contains extra arguments to be used 142 # by sphinx-build time. Among them, it may contain sphinx verbosity 143 # and desired number of parallel jobs. 144 # 145 parser = argparse.ArgumentParser() 146 parser.add_argument('-j', '--jobs', type=int) 147 parser.add_argument('-q', '--quiet', type=int) 148 149 # 150 # Other sphinx-build arguments go as-is, so place them 151 # at self.sphinxopts, using shell parser 152 # 153 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 154 155 # 156 # Build a list of sphinx args, honoring verbosity here if specified 157 # 158 159 verbose = self.verbose 160 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 161 if sphinx_args.quiet is True: 162 verbose = False 163 164 # 165 # If the user explicitly sets "-j" at command line, use it. 166 # Otherwise, pick it from SPHINXOPTS args 167 # 168 if n_jobs: 169 self.n_jobs = n_jobs 170 elif sphinx_args.jobs: 171 self.n_jobs = sphinx_args.jobs 172 else: 173 self.n_jobs = None 174 175 if not verbose: 176 self.sphinxopts += ["-q"] 177 178 def __init__(self, builddir, verbose=False, n_jobs=None, interactive=None): 179 """Initialize internal variables""" 180 self.verbose = None 181 182 # 183 # Normal variables passed from Kernel's makefile 184 # 185 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 186 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 187 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 188 189 if not interactive: 190 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 191 else: 192 self.latexopts = os.environ.get("LATEXOPTS", "") 193 194 if not verbose: 195 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 196 197 if verbose is not None: 198 self.verbose = verbose 199 200 # 201 # Source tree directory. This needs to be at os.environ, as 202 # Sphinx extensions use it 203 # 204 self.srctree = os.environ.get("srctree") 205 if not self.srctree: 206 self.srctree = "." 207 os.environ["srctree"] = self.srctree 208 209 # 210 # Now that we can expand srctree, get other directories as well 211 # 212 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 213 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 214 "scripts/kernel-doc.py")) 215 self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) 216 217 self.config_rust = self.is_rust_enabled() 218 219 # 220 # Get directory locations for LaTeX build toolchain 221 # 222 self.pdflatex_cmd = shutil.which(self.pdflatex) 223 self.latexmk_cmd = shutil.which("latexmk") 224 225 self.env = os.environ.copy() 226 227 self.get_sphinx_extra_opts(n_jobs) 228 229 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 230 """ 231 Executes sphinx-build using current python3 command. 232 233 When calling via GNU make, POSIX jobserver is used to tell how 234 many jobs are still available from a job pool. claim all remaining 235 jobs, as we don't want sphinx-build to run in parallel with other 236 jobs. 237 238 Despite that, the user may actually force a different value than 239 the number of available jobs via command line. 240 241 The "with" logic here is used to ensure that the claimed jobs will 242 be freed once subprocess finishes 243 """ 244 245 with JobserverExec() as jobserver: 246 if jobserver.claim: 247 # 248 # when GNU make is used, claim available jobs from jobserver 249 # 250 n_jobs = str(jobserver.claim) 251 else: 252 # 253 # Otherwise, let sphinx decide by default 254 # 255 n_jobs = "auto" 256 257 # 258 # If explicitly requested via command line, override default 259 # 260 if self.n_jobs: 261 n_jobs = str(self.n_jobs) 262 263 cmd = [sys.executable, sphinx_build] 264 cmd += [f"-j{n_jobs}"] 265 cmd += self.sphinxopts 266 cmd += build_args 267 268 if self.verbose: 269 print(" ".join(cmd)) 270 271 return subprocess.call(cmd, *args, **pwargs) 272 273 def handle_html(self, css, output_dir): 274 """ 275 Extra steps for HTML and epub output. 276 277 For such targets, we need to ensure that CSS will be properly 278 copied to the output _static directory 279 """ 280 281 if not css: 282 return 283 284 css = os.path.expanduser(css) 285 if not css.startswith("/"): 286 css = os.path.join(self.srctree, css) 287 288 static_dir = os.path.join(output_dir, "_static") 289 os.makedirs(static_dir, exist_ok=True) 290 291 try: 292 shutil.copy2(css, static_dir) 293 except (OSError, IOError) as e: 294 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 295 296 def build_pdf_file(self, latex_cmd, from_dir, path): 297 """Builds a single pdf file using latex_cmd""" 298 try: 299 subprocess.run(latex_cmd + [path], 300 cwd=from_dir, check=True, env=self.env) 301 302 return True 303 except subprocess.CalledProcessError: 304 return False 305 306 def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): 307 """Build PDF files in parallel if possible""" 308 builds = {} 309 build_failed = False 310 max_len = 0 311 has_tex = False 312 313 # 314 # LaTeX PDF error code is almost useless for us: 315 # any warning makes it non-zero. For kernel doc builds it always return 316 # non-zero even when build succeeds. So, let's do the best next thing: 317 # Ignore build errors. At the end, check if all PDF files were built, 318 # printing a summary with the built ones and returning 0 if all of 319 # them were actually built. 320 # 321 with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: 322 jobs = {} 323 324 for from_dir, pdf_dir, entry in tex_files: 325 name = entry.name 326 327 if not name.endswith(tex_suffix): 328 continue 329 330 name = name[:-len(tex_suffix)] 331 332 max_len = max(max_len, len(name)) 333 334 has_tex = True 335 336 future = executor.submit(self.build_pdf_file, latex_cmd, 337 from_dir, entry.path) 338 jobs[future] = (from_dir, pdf_dir, name) 339 340 for future in futures.as_completed(jobs): 341 from_dir, pdf_dir, name = jobs[future] 342 343 pdf_name = name + ".pdf" 344 pdf_from = os.path.join(from_dir, pdf_name) 345 346 try: 347 success = future.result() 348 349 if success and os.path.exists(pdf_from): 350 pdf_to = os.path.join(pdf_dir, pdf_name) 351 352 os.rename(pdf_from, pdf_to) 353 354 # 355 # if verbose, get the name of built PDF file 356 # 357 if self.verbose: 358 builds[name] = os.path.relpath(pdf_to, self.builddir) 359 else: 360 builds[name] = "FAILED" 361 build_failed = True 362 except futures.Error as e: 363 builds[name] = f"FAILED ({repr(e)})" 364 build_failed = True 365 366 # 367 # Handle case where no .tex files were found 368 # 369 if not has_tex: 370 name = "Sphinx LaTeX builder" 371 max_len = max(max_len, len(name)) 372 builds[name] = "FAILED (no .tex file was generated)" 373 build_failed = True 374 375 return builds, build_failed, max_len 376 377 def handle_pdf(self, output_dirs, deny_vf): 378 """ 379 Extra steps for PDF output. 380 381 As PDF is handled via a LaTeX output, after building the .tex file, 382 a new build is needed to create the PDF output from the latex 383 directory. 384 """ 385 builds = {} 386 max_len = 0 387 tex_suffix = ".tex" 388 tex_files = [] 389 390 # 391 # Since early 2024, Fedora and openSUSE tumbleweed have started 392 # deploying variable-font format of "Noto CJK", causing LaTeX 393 # to break with CJK. Work around it, by denying the variable font 394 # usage during xelatex build by passing the location of a config 395 # file with a deny list. 396 # 397 # See tools/docs/lib/latex_fonts.py for more details. 398 # 399 if deny_vf: 400 deny_vf = os.path.expanduser(deny_vf) 401 if os.path.isdir(deny_vf): 402 self.env["XDG_CONFIG_HOME"] = deny_vf 403 404 for from_dir in output_dirs: 405 pdf_dir = os.path.join(from_dir, "../pdf") 406 os.makedirs(pdf_dir, exist_ok=True) 407 408 if self.latexmk_cmd: 409 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 410 else: 411 latex_cmd = [self.pdflatex] 412 413 latex_cmd.extend(shlex.split(self.latexopts)) 414 415 # Get a list of tex files to process 416 with os.scandir(from_dir) as it: 417 for entry in it: 418 if entry.name.endswith(tex_suffix): 419 tex_files.append((from_dir, pdf_dir, entry)) 420 421 # 422 # When using make, this won't be used, as the number of jobs comes 423 # from POSIX jobserver. So, this covers the case where build comes 424 # from command line. On such case, serialize by default, except if 425 # the user explicitly sets the number of jobs. 426 # 427 n_jobs = 1 428 429 # n_jobs is either an integer or "auto". Only use it if it is a number 430 if self.n_jobs: 431 try: 432 n_jobs = int(self.n_jobs) 433 except ValueError: 434 pass 435 436 # 437 # When using make, jobserver.claim is the number of jobs that were 438 # used with "-j" and that aren't used by other make targets 439 # 440 with JobserverExec() as jobserver: 441 n_jobs = 1 442 443 # 444 # Handle the case when a parameter is passed via command line, 445 # using it as default, if jobserver doesn't claim anything 446 # 447 if self.n_jobs: 448 try: 449 n_jobs = int(self.n_jobs) 450 except ValueError: 451 pass 452 453 if jobserver.claim: 454 n_jobs = jobserver.claim 455 456 builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, 457 latex_cmd, 458 tex_files, 459 n_jobs) 460 461 # 462 # In verbose mode, print a summary with the build results per file. 463 # Otherwise, print a single line with all failures, if any. 464 # On both cases, return code 1 indicates build failures, 465 # 466 if self.verbose: 467 msg = "Summary" 468 msg += "\n" + "=" * len(msg) 469 print() 470 print(msg) 471 472 for pdf_name, pdf_file in builds.items(): 473 print(f"{pdf_name:<{max_len}}: {pdf_file}") 474 475 print() 476 if build_failed: 477 msg = LatexFontChecker().check() 478 if msg: 479 print(msg) 480 481 sys.exit("Error: not all PDF files were created.") 482 483 elif build_failed: 484 n_failures = len(builds) 485 failures = ", ".join(builds.keys()) 486 487 msg = LatexFontChecker().check() 488 if msg: 489 print(msg) 490 491 sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") 492 493 def handle_info(self, output_dirs): 494 """ 495 Extra steps for Info output. 496 497 For texinfo generation, an additional make is needed from the 498 texinfo directory. 499 """ 500 501 for output_dir in output_dirs: 502 try: 503 subprocess.run(["make", "info"], cwd=output_dir, check=True) 504 except subprocess.CalledProcessError as e: 505 sys.exit(f"Error generating info docs: {e}") 506 507 def cleandocs(self, builder): # pylint: disable=W0613 508 """Remove documentation output directory""" 509 shutil.rmtree(self.builddir, ignore_errors=True) 510 511 def build(self, target, sphinxdirs=None, conf="conf.py", 512 theme=None, css=None, paper=None, deny_vf=None): 513 """ 514 Build documentation using Sphinx. This is the core function of this 515 module. It prepares all arguments required by sphinx-build. 516 """ 517 518 builder = TARGETS[target]["builder"] 519 out_dir = TARGETS[target].get("out_dir", "") 520 521 # 522 # Cleandocs doesn't require sphinx-build 523 # 524 if target == "cleandocs": 525 self.cleandocs(builder) 526 return 527 528 if theme: 529 os.environ["DOCS_THEME"] = theme 530 531 # 532 # Other targets require sphinx-build, so check if it exists 533 # 534 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 535 if not sphinxbuild: 536 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 537 538 if builder == "latex": 539 if not self.pdflatex_cmd and not self.latexmk_cmd: 540 sys.exit("Error: pdflatex or latexmk required for PDF generation") 541 542 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 543 544 # 545 # Fill in base arguments for Sphinx build 546 # 547 kerneldoc = self.kerneldoc 548 if kerneldoc.startswith(self.srctree): 549 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 550 551 args = [ "-b", builder, "-c", docs_dir ] 552 553 if builder == "latex": 554 if not paper: 555 paper = PAPER[1] 556 557 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 558 559 if self.config_rust: 560 args.extend(["-t", "rustdoc"]) 561 562 if conf: 563 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 564 565 if not sphinxdirs: 566 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 567 568 # 569 # sphinxdirs can be a list or a whitespace-separated string 570 # 571 sphinxdirs_list = [] 572 for sphinxdir in sphinxdirs: 573 if isinstance(sphinxdir, list): 574 sphinxdirs_list += sphinxdir 575 else: 576 sphinxdirs_list += sphinxdir.split() 577 578 # 579 # Step 1: Build each directory in separate. 580 # 581 # This is not the best way of handling it, as cross-references between 582 # them will be broken, but this is what we've been doing since 583 # the beginning. 584 # 585 output_dirs = [] 586 for sphinxdir in sphinxdirs_list: 587 src_dir = os.path.join(docs_dir, sphinxdir) 588 doctree_dir = os.path.join(self.builddir, ".doctrees") 589 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 590 591 # 592 # Make directory names canonical 593 # 594 src_dir = os.path.normpath(src_dir) 595 doctree_dir = os.path.normpath(doctree_dir) 596 output_dir = os.path.normpath(output_dir) 597 598 os.makedirs(doctree_dir, exist_ok=True) 599 os.makedirs(output_dir, exist_ok=True) 600 601 output_dirs.append(output_dir) 602 603 build_args = args + [ 604 "-d", doctree_dir, 605 "-D", f"kerneldoc_bin={kerneldoc}", 606 "-D", f"version={self.kernelversion}", 607 "-D", f"release={self.kernelrelease}", 608 "-D", f"kerneldoc_srctree={self.srctree}", 609 src_dir, 610 output_dir, 611 ] 612 613 try: 614 self.run_sphinx(sphinxbuild, build_args, env=self.env) 615 except (OSError, ValueError, subprocess.SubprocessError) as e: 616 sys.exit(f"Build failed: {repr(e)}") 617 618 # 619 # Ensure that each html/epub output will have needed static files 620 # 621 if target in ["htmldocs", "epubdocs"]: 622 self.handle_html(css, output_dir) 623 624 # 625 # Step 2: Some targets (PDF and info) require an extra step once 626 # sphinx-build finishes 627 # 628 if target == "pdfdocs": 629 self.handle_pdf(output_dirs, deny_vf) 630 elif target == "infodocs": 631 self.handle_info(output_dirs) 632 633def jobs_type(value): 634 """ 635 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 636 equal or bigger than one. 637 """ 638 if value is None: 639 return None 640 641 if value.lower() == 'auto': 642 return value.lower() 643 644 try: 645 if int(value) >= 1: 646 return value 647 648 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 649 except ValueError: 650 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 651 652def main(): 653 """ 654 Main function. The only mandatory argument is the target. If not 655 specified, the other arguments will use default values if not 656 specified at os.environ. 657 """ 658 parser = argparse.ArgumentParser(description="Kernel documentation builder") 659 660 parser.add_argument("target", choices=list(TARGETS.keys()), 661 help="Documentation target to build") 662 parser.add_argument("--sphinxdirs", nargs="+", 663 help="Specific directories to build") 664 parser.add_argument("--conf", default="conf.py", 665 help="Sphinx configuration file") 666 parser.add_argument("--builddir", default="output", 667 help="Sphinx configuration file") 668 669 parser.add_argument("--theme", help="Sphinx theme to use") 670 671 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 672 673 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 674 help="Paper size for LaTeX/PDF output") 675 676 parser.add_argument('--deny-vf', 677 help="Configuration to deny variable fonts on pdf builds") 678 679 parser.add_argument("-v", "--verbose", action='store_true', 680 help="place build in verbose mode") 681 682 parser.add_argument('-j', '--jobs', type=jobs_type, 683 help="Sets number of jobs to use with sphinx-build") 684 685 parser.add_argument('-i', '--interactive', action='store_true', 686 help="Change latex default to run in interactive mode") 687 688 args = parser.parse_args() 689 690 PythonVersion.check_python(MIN_PYTHON_VERSION) 691 692 builder = SphinxBuilder(builddir=args.builddir, 693 verbose=args.verbose, n_jobs=args.jobs, 694 interactive=args.interactive) 695 696 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 697 theme=args.theme, css=args.css, paper=args.paper, 698 deny_vf=args.deny_vf) 699 700if __name__ == "__main__": 701 main() 702