1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import locale 49import os 50import shlex 51import shutil 52import subprocess 53import sys 54 55from concurrent import futures 56 57from lib.python_version import PythonVersion 58from lib.latex_fonts import LatexFontChecker 59 60LIB_DIR = "../../scripts/lib" 61SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 62 63sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 64 65from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 66 67# 68# Some constants 69# 70MIN_PYTHON_VERSION = PythonVersion("3.7").version 71PAPER = ["", "a4", "letter"] 72 73TARGETS = { 74 "cleandocs": { "builder": "clean" }, 75 "linkcheckdocs": { "builder": "linkcheck" }, 76 "htmldocs": { "builder": "html" }, 77 "epubdocs": { "builder": "epub", "out_dir": "epub" }, 78 "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 79 "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 80 "latexdocs": { "builder": "latex", "out_dir": "latex" }, 81 "pdfdocs": { "builder": "latex", "out_dir": "latex" }, 82 "xmldocs": { "builder": "xml", "out_dir": "xml" }, 83} 84 85 86# 87# SphinxBuilder class 88# 89 90class SphinxBuilder: 91 """ 92 Handles a sphinx-build target, adding needed arguments to build 93 with the Kernel. 94 """ 95 96 def is_rust_enabled(self): 97 """Check if rust is enabled at .config""" 98 config_path = os.path.join(self.srctree, ".config") 99 if os.path.isfile(config_path): 100 with open(config_path, "r", encoding="utf-8") as f: 101 return "CONFIG_RUST=y" in f.read() 102 return False 103 104 def get_path(self, path, use_cwd=False, abs_path=False): 105 """ 106 Ancillary routine to handle patches the right way, as shell does. 107 108 It first expands "~" and "~user". Then, if patch is not absolute, 109 join self.srctree. Finally, if requested, convert to abspath. 110 """ 111 112 path = os.path.expanduser(path) 113 if not path.startswith("/"): 114 if use_cwd: 115 base = os.getcwd() 116 else: 117 base = self.srctree 118 119 path = os.path.join(base, path) 120 121 if abs_path: 122 return os.path.abspath(path) 123 124 return path 125 126 def get_sphinx_extra_opts(self, n_jobs): 127 """ 128 Get the number of jobs to be used for docs build passed via command 129 line and desired sphinx verbosity. 130 131 The number of jobs can be on different places: 132 133 1) It can be passed via "-j" argument; 134 2) The SPHINXOPTS="-j8" env var may have "-j"; 135 3) if called via GNU make, -j specifies the desired number of jobs. 136 with GNU makefile, this number is available via POSIX jobserver; 137 4) if none of the above is available, it should default to "-jauto", 138 and let sphinx decide the best value. 139 """ 140 141 # 142 # SPHINXOPTS env var, if used, contains extra arguments to be used 143 # by sphinx-build time. Among them, it may contain sphinx verbosity 144 # and desired number of parallel jobs. 145 # 146 parser = argparse.ArgumentParser() 147 parser.add_argument('-j', '--jobs', type=int) 148 parser.add_argument('-q', '--quiet', type=int) 149 150 # 151 # Other sphinx-build arguments go as-is, so place them 152 # at self.sphinxopts, using shell parser 153 # 154 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 155 156 # 157 # Build a list of sphinx args, honoring verbosity here if specified 158 # 159 160 verbose = self.verbose 161 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 162 if sphinx_args.quiet is True: 163 verbose = False 164 165 # 166 # If the user explicitly sets "-j" at command line, use it. 167 # Otherwise, pick it from SPHINXOPTS args 168 # 169 if n_jobs: 170 self.n_jobs = n_jobs 171 elif sphinx_args.jobs: 172 self.n_jobs = sphinx_args.jobs 173 else: 174 self.n_jobs = None 175 176 if not verbose: 177 self.sphinxopts += ["-q"] 178 179 def __init__(self, builddir, verbose=False, n_jobs=None, interactive=None): 180 """Initialize internal variables""" 181 self.verbose = None 182 183 # 184 # Normal variables passed from Kernel's makefile 185 # 186 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 187 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 188 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 189 190 if not interactive: 191 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 192 else: 193 self.latexopts = os.environ.get("LATEXOPTS", "") 194 195 if not verbose: 196 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 197 198 if verbose is not None: 199 self.verbose = verbose 200 201 # 202 # Source tree directory. This needs to be at os.environ, as 203 # Sphinx extensions use it 204 # 205 self.srctree = os.environ.get("srctree") 206 if not self.srctree: 207 self.srctree = "." 208 os.environ["srctree"] = self.srctree 209 210 # 211 # Now that we can expand srctree, get other directories as well 212 # 213 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 214 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 215 "scripts/kernel-doc.py")) 216 self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) 217 218 self.config_rust = self.is_rust_enabled() 219 220 # 221 # Get directory locations for LaTeX build toolchain 222 # 223 self.pdflatex_cmd = shutil.which(self.pdflatex) 224 self.latexmk_cmd = shutil.which("latexmk") 225 226 self.env = os.environ.copy() 227 228 self.get_sphinx_extra_opts(n_jobs) 229 230 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 231 """ 232 Executes sphinx-build using current python3 command. 233 234 When calling via GNU make, POSIX jobserver is used to tell how 235 many jobs are still available from a job pool. claim all remaining 236 jobs, as we don't want sphinx-build to run in parallel with other 237 jobs. 238 239 Despite that, the user may actually force a different value than 240 the number of available jobs via command line. 241 242 The "with" logic here is used to ensure that the claimed jobs will 243 be freed once subprocess finishes 244 """ 245 246 with JobserverExec() as jobserver: 247 if jobserver.claim: 248 # 249 # when GNU make is used, claim available jobs from jobserver 250 # 251 n_jobs = str(jobserver.claim) 252 else: 253 # 254 # Otherwise, let sphinx decide by default 255 # 256 n_jobs = "auto" 257 258 # 259 # If explicitly requested via command line, override default 260 # 261 if self.n_jobs: 262 n_jobs = str(self.n_jobs) 263 264 cmd = [sys.executable, sphinx_build] 265 cmd += [f"-j{n_jobs}"] 266 cmd += self.sphinxopts 267 cmd += build_args 268 269 if self.verbose: 270 print(" ".join(cmd)) 271 272 return subprocess.call(cmd, *args, **pwargs) 273 274 def handle_html(self, css, output_dir): 275 """ 276 Extra steps for HTML and epub output. 277 278 For such targets, we need to ensure that CSS will be properly 279 copied to the output _static directory 280 """ 281 282 if not css: 283 return 284 285 css = os.path.expanduser(css) 286 if not css.startswith("/"): 287 css = os.path.join(self.srctree, css) 288 289 static_dir = os.path.join(output_dir, "_static") 290 os.makedirs(static_dir, exist_ok=True) 291 292 try: 293 shutil.copy2(css, static_dir) 294 except (OSError, IOError) as e: 295 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 296 297 def build_pdf_file(self, latex_cmd, from_dir, path): 298 """Builds a single pdf file using latex_cmd""" 299 try: 300 subprocess.run(latex_cmd + [path], 301 cwd=from_dir, check=True, env=self.env) 302 303 return True 304 except subprocess.CalledProcessError: 305 return False 306 307 def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): 308 """Build PDF files in parallel if possible""" 309 builds = {} 310 build_failed = False 311 max_len = 0 312 has_tex = False 313 314 # 315 # LaTeX PDF error code is almost useless for us: 316 # any warning makes it non-zero. For kernel doc builds it always return 317 # non-zero even when build succeeds. So, let's do the best next thing: 318 # Ignore build errors. At the end, check if all PDF files were built, 319 # printing a summary with the built ones and returning 0 if all of 320 # them were actually built. 321 # 322 with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: 323 jobs = {} 324 325 for from_dir, pdf_dir, entry in tex_files: 326 name = entry.name 327 328 if not name.endswith(tex_suffix): 329 continue 330 331 name = name[:-len(tex_suffix)] 332 333 max_len = max(max_len, len(name)) 334 335 has_tex = True 336 337 future = executor.submit(self.build_pdf_file, latex_cmd, 338 from_dir, entry.path) 339 jobs[future] = (from_dir, pdf_dir, name) 340 341 for future in futures.as_completed(jobs): 342 from_dir, pdf_dir, name = jobs[future] 343 344 pdf_name = name + ".pdf" 345 pdf_from = os.path.join(from_dir, pdf_name) 346 347 try: 348 success = future.result() 349 350 if success and os.path.exists(pdf_from): 351 pdf_to = os.path.join(pdf_dir, pdf_name) 352 353 os.rename(pdf_from, pdf_to) 354 355 # 356 # if verbose, get the name of built PDF file 357 # 358 if self.verbose: 359 builds[name] = os.path.relpath(pdf_to, self.builddir) 360 else: 361 builds[name] = "FAILED" 362 build_failed = True 363 except futures.Error as e: 364 builds[name] = f"FAILED ({repr(e)})" 365 build_failed = True 366 367 # 368 # Handle case where no .tex files were found 369 # 370 if not has_tex: 371 name = "Sphinx LaTeX builder" 372 max_len = max(max_len, len(name)) 373 builds[name] = "FAILED (no .tex file was generated)" 374 build_failed = True 375 376 return builds, build_failed, max_len 377 378 def handle_pdf(self, output_dirs, deny_vf): 379 """ 380 Extra steps for PDF output. 381 382 As PDF is handled via a LaTeX output, after building the .tex file, 383 a new build is needed to create the PDF output from the latex 384 directory. 385 """ 386 builds = {} 387 max_len = 0 388 tex_suffix = ".tex" 389 tex_files = [] 390 391 # 392 # Since early 2024, Fedora and openSUSE tumbleweed have started 393 # deploying variable-font format of "Noto CJK", causing LaTeX 394 # to break with CJK. Work around it, by denying the variable font 395 # usage during xelatex build by passing the location of a config 396 # file with a deny list. 397 # 398 # See tools/docs/lib/latex_fonts.py for more details. 399 # 400 if deny_vf: 401 deny_vf = os.path.expanduser(deny_vf) 402 if os.path.isdir(deny_vf): 403 self.env["XDG_CONFIG_HOME"] = deny_vf 404 405 for from_dir in output_dirs: 406 pdf_dir = os.path.join(from_dir, "../pdf") 407 os.makedirs(pdf_dir, exist_ok=True) 408 409 if self.latexmk_cmd: 410 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 411 else: 412 latex_cmd = [self.pdflatex] 413 414 latex_cmd.extend(shlex.split(self.latexopts)) 415 416 # Get a list of tex files to process 417 with os.scandir(from_dir) as it: 418 for entry in it: 419 if entry.name.endswith(tex_suffix): 420 tex_files.append((from_dir, pdf_dir, entry)) 421 422 # 423 # When using make, this won't be used, as the number of jobs comes 424 # from POSIX jobserver. So, this covers the case where build comes 425 # from command line. On such case, serialize by default, except if 426 # the user explicitly sets the number of jobs. 427 # 428 n_jobs = 1 429 430 # n_jobs is either an integer or "auto". Only use it if it is a number 431 if self.n_jobs: 432 try: 433 n_jobs = int(self.n_jobs) 434 except ValueError: 435 pass 436 437 # 438 # When using make, jobserver.claim is the number of jobs that were 439 # used with "-j" and that aren't used by other make targets 440 # 441 with JobserverExec() as jobserver: 442 n_jobs = 1 443 444 # 445 # Handle the case when a parameter is passed via command line, 446 # using it as default, if jobserver doesn't claim anything 447 # 448 if self.n_jobs: 449 try: 450 n_jobs = int(self.n_jobs) 451 except ValueError: 452 pass 453 454 if jobserver.claim: 455 n_jobs = jobserver.claim 456 457 builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, 458 latex_cmd, 459 tex_files, 460 n_jobs) 461 462 # 463 # In verbose mode, print a summary with the build results per file. 464 # Otherwise, print a single line with all failures, if any. 465 # On both cases, return code 1 indicates build failures, 466 # 467 if self.verbose: 468 msg = "Summary" 469 msg += "\n" + "=" * len(msg) 470 print() 471 print(msg) 472 473 for pdf_name, pdf_file in builds.items(): 474 print(f"{pdf_name:<{max_len}}: {pdf_file}") 475 476 print() 477 if build_failed: 478 msg = LatexFontChecker().check() 479 if msg: 480 print(msg) 481 482 sys.exit("Error: not all PDF files were created.") 483 484 elif build_failed: 485 n_failures = len(builds) 486 failures = ", ".join(builds.keys()) 487 488 msg = LatexFontChecker().check() 489 if msg: 490 print(msg) 491 492 sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") 493 494 def handle_info(self, output_dirs): 495 """ 496 Extra steps for Info output. 497 498 For texinfo generation, an additional make is needed from the 499 texinfo directory. 500 """ 501 502 for output_dir in output_dirs: 503 try: 504 subprocess.run(["make", "info"], cwd=output_dir, check=True) 505 except subprocess.CalledProcessError as e: 506 sys.exit(f"Error generating info docs: {e}") 507 508 def cleandocs(self, builder): # pylint: disable=W0613 509 """Remove documentation output directory""" 510 shutil.rmtree(self.builddir, ignore_errors=True) 511 512 def build(self, target, sphinxdirs=None, conf="conf.py", 513 theme=None, css=None, paper=None, deny_vf=None): 514 """ 515 Build documentation using Sphinx. This is the core function of this 516 module. It prepares all arguments required by sphinx-build. 517 """ 518 519 builder = TARGETS[target]["builder"] 520 out_dir = TARGETS[target].get("out_dir", "") 521 522 # 523 # Cleandocs doesn't require sphinx-build 524 # 525 if target == "cleandocs": 526 self.cleandocs(builder) 527 return 528 529 if theme: 530 os.environ["DOCS_THEME"] = theme 531 532 # 533 # Other targets require sphinx-build, so check if it exists 534 # 535 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 536 if not sphinxbuild: 537 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 538 539 if builder == "latex": 540 if not self.pdflatex_cmd and not self.latexmk_cmd: 541 sys.exit("Error: pdflatex or latexmk required for PDF generation") 542 543 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 544 545 # 546 # Fill in base arguments for Sphinx build 547 # 548 kerneldoc = self.kerneldoc 549 if kerneldoc.startswith(self.srctree): 550 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 551 552 args = [ "-b", builder, "-c", docs_dir ] 553 554 if builder == "latex": 555 if not paper: 556 paper = PAPER[1] 557 558 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 559 560 if self.config_rust: 561 args.extend(["-t", "rustdoc"]) 562 563 if conf: 564 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 565 566 if not sphinxdirs: 567 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 568 569 # 570 # The sphinx-build tool has a bug: internally, it tries to set 571 # locale with locale.setlocale(locale.LC_ALL, ''). This causes a 572 # crash if language is not set. Detect and fix it. 573 # 574 try: 575 locale.setlocale(locale.LC_ALL, '') 576 except locale.Error: 577 self.env["LC_ALL"] = "C" 578 579 # 580 # sphinxdirs can be a list or a whitespace-separated string 581 # 582 sphinxdirs_list = [] 583 for sphinxdir in sphinxdirs: 584 if isinstance(sphinxdir, list): 585 sphinxdirs_list += sphinxdir 586 else: 587 sphinxdirs_list += sphinxdir.split() 588 589 # 590 # Step 1: Build each directory in separate. 591 # 592 # This is not the best way of handling it, as cross-references between 593 # them will be broken, but this is what we've been doing since 594 # the beginning. 595 # 596 output_dirs = [] 597 for sphinxdir in sphinxdirs_list: 598 src_dir = os.path.join(docs_dir, sphinxdir) 599 doctree_dir = os.path.join(self.builddir, ".doctrees") 600 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 601 602 # 603 # Make directory names canonical 604 # 605 src_dir = os.path.normpath(src_dir) 606 doctree_dir = os.path.normpath(doctree_dir) 607 output_dir = os.path.normpath(output_dir) 608 609 os.makedirs(doctree_dir, exist_ok=True) 610 os.makedirs(output_dir, exist_ok=True) 611 612 output_dirs.append(output_dir) 613 614 build_args = args + [ 615 "-d", doctree_dir, 616 "-D", f"kerneldoc_bin={kerneldoc}", 617 "-D", f"version={self.kernelversion}", 618 "-D", f"release={self.kernelrelease}", 619 "-D", f"kerneldoc_srctree={self.srctree}", 620 src_dir, 621 output_dir, 622 ] 623 624 try: 625 self.run_sphinx(sphinxbuild, build_args, env=self.env) 626 except (OSError, ValueError, subprocess.SubprocessError) as e: 627 sys.exit(f"Build failed: {repr(e)}") 628 629 # 630 # Ensure that each html/epub output will have needed static files 631 # 632 if target in ["htmldocs", "epubdocs"]: 633 self.handle_html(css, output_dir) 634 635 # 636 # Step 2: Some targets (PDF and info) require an extra step once 637 # sphinx-build finishes 638 # 639 if target == "pdfdocs": 640 self.handle_pdf(output_dirs, deny_vf) 641 elif target == "infodocs": 642 self.handle_info(output_dirs) 643 644def jobs_type(value): 645 """ 646 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 647 equal or bigger than one. 648 """ 649 if value is None: 650 return None 651 652 if value.lower() == 'auto': 653 return value.lower() 654 655 try: 656 if int(value) >= 1: 657 return value 658 659 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 660 except ValueError: 661 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 662 663def main(): 664 """ 665 Main function. The only mandatory argument is the target. If not 666 specified, the other arguments will use default values if not 667 specified at os.environ. 668 """ 669 parser = argparse.ArgumentParser(description="Kernel documentation builder") 670 671 parser.add_argument("target", choices=list(TARGETS.keys()), 672 help="Documentation target to build") 673 parser.add_argument("--sphinxdirs", nargs="+", 674 help="Specific directories to build") 675 parser.add_argument("--conf", default="conf.py", 676 help="Sphinx configuration file") 677 parser.add_argument("--builddir", default="output", 678 help="Sphinx configuration file") 679 680 parser.add_argument("--theme", help="Sphinx theme to use") 681 682 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 683 684 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 685 help="Paper size for LaTeX/PDF output") 686 687 parser.add_argument('--deny-vf', 688 help="Configuration to deny variable fonts on pdf builds") 689 690 parser.add_argument("-v", "--verbose", action='store_true', 691 help="place build in verbose mode") 692 693 parser.add_argument('-j', '--jobs', type=jobs_type, 694 help="Sets number of jobs to use with sphinx-build") 695 696 parser.add_argument('-i', '--interactive', action='store_true', 697 help="Change latex default to run in interactive mode") 698 699 args = parser.parse_args() 700 701 PythonVersion.check_python(MIN_PYTHON_VERSION) 702 703 builder = SphinxBuilder(builddir=args.builddir, 704 verbose=args.verbose, n_jobs=args.jobs, 705 interactive=args.interactive) 706 707 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 708 theme=args.theme, css=args.css, paper=args.paper, 709 deny_vf=args.deny_vf) 710 711if __name__ == "__main__": 712 main() 713