1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import locale 49import os 50import shlex 51import shutil 52import subprocess 53import sys 54 55from concurrent import futures 56 57from lib.python_version import PythonVersion 58from lib.latex_fonts import LatexFontChecker 59 60LIB_DIR = "../../scripts/lib" 61SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 62 63sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 64 65from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 66 67# 68# Some constants 69# 70MIN_PYTHON_VERSION = PythonVersion("3.7").version 71PAPER = ["", "a4", "letter"] 72 73TARGETS = { 74 "cleandocs": { "builder": "clean" }, 75 "linkcheckdocs": { "builder": "linkcheck" }, 76 "htmldocs": { "builder": "html" }, 77 "epubdocs": { "builder": "epub", "out_dir": "epub" }, 78 "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 79 "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 80 "latexdocs": { "builder": "latex", "out_dir": "latex" }, 81 "pdfdocs": { "builder": "latex", "out_dir": "latex" }, 82 "xmldocs": { "builder": "xml", "out_dir": "xml" }, 83} 84 85 86# 87# SphinxBuilder class 88# 89 90class SphinxBuilder: 91 """ 92 Handles a sphinx-build target, adding needed arguments to build 93 with the Kernel. 94 """ 95 96 def is_rust_enabled(self): 97 """Check if rust is enabled at .config""" 98 config_path = os.path.join(self.srctree, ".config") 99 if os.path.isfile(config_path): 100 with open(config_path, "r", encoding="utf-8") as f: 101 return "CONFIG_RUST=y" in f.read() 102 return False 103 104 def get_path(self, path, use_cwd=False, abs_path=False): 105 """ 106 Ancillary routine to handle patches the right way, as shell does. 107 108 It first expands "~" and "~user". Then, if patch is not absolute, 109 join self.srctree. Finally, if requested, convert to abspath. 110 """ 111 112 path = os.path.expanduser(path) 113 if not path.startswith("/"): 114 if use_cwd: 115 base = os.getcwd() 116 else: 117 base = self.srctree 118 119 path = os.path.join(base, path) 120 121 if abs_path: 122 return os.path.abspath(path) 123 124 return path 125 126 def get_sphinx_extra_opts(self, n_jobs): 127 """ 128 Get the number of jobs to be used for docs build passed via command 129 line and desired sphinx verbosity. 130 131 The number of jobs can be on different places: 132 133 1) It can be passed via "-j" argument; 134 2) The SPHINXOPTS="-j8" env var may have "-j"; 135 3) if called via GNU make, -j specifies the desired number of jobs. 136 with GNU makefile, this number is available via POSIX jobserver; 137 4) if none of the above is available, it should default to "-jauto", 138 and let sphinx decide the best value. 139 """ 140 141 # 142 # SPHINXOPTS env var, if used, contains extra arguments to be used 143 # by sphinx-build time. Among them, it may contain sphinx verbosity 144 # and desired number of parallel jobs. 145 # 146 parser = argparse.ArgumentParser() 147 parser.add_argument('-j', '--jobs', type=int) 148 parser.add_argument('-q', '--quiet', type=int) 149 150 # 151 # Other sphinx-build arguments go as-is, so place them 152 # at self.sphinxopts, using shell parser 153 # 154 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 155 156 # 157 # Build a list of sphinx args, honoring verbosity here if specified 158 # 159 160 verbose = self.verbose 161 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 162 if sphinx_args.quiet is True: 163 verbose = False 164 165 # 166 # If the user explicitly sets "-j" at command line, use it. 167 # Otherwise, pick it from SPHINXOPTS args 168 # 169 if n_jobs: 170 self.n_jobs = n_jobs 171 elif sphinx_args.jobs: 172 self.n_jobs = sphinx_args.jobs 173 else: 174 self.n_jobs = None 175 176 if not verbose: 177 self.sphinxopts += ["-q"] 178 179 def __init__(self, builddir, verbose=False, n_jobs=None, interactive=None): 180 """Initialize internal variables""" 181 self.verbose = None 182 183 # 184 # Normal variables passed from Kernel's makefile 185 # 186 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 187 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 188 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 189 190 if not interactive: 191 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 192 else: 193 self.latexopts = os.environ.get("LATEXOPTS", "") 194 195 if not verbose: 196 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 197 198 if verbose is not None: 199 self.verbose = verbose 200 201 # 202 # Source tree directory. This needs to be at os.environ, as 203 # Sphinx extensions use it 204 # 205 self.srctree = os.environ.get("srctree") 206 if not self.srctree: 207 self.srctree = "." 208 os.environ["srctree"] = self.srctree 209 210 # 211 # Now that we can expand srctree, get other directories as well 212 # 213 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 214 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 215 "scripts/kernel-doc.py")) 216 self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) 217 218 self.config_rust = self.is_rust_enabled() 219 220 # 221 # Get directory locations for LaTeX build toolchain 222 # 223 self.pdflatex_cmd = shutil.which(self.pdflatex) 224 self.latexmk_cmd = shutil.which("latexmk") 225 226 self.env = os.environ.copy() 227 228 self.get_sphinx_extra_opts(n_jobs) 229 230 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 231 """ 232 Executes sphinx-build using current python3 command. 233 234 When calling via GNU make, POSIX jobserver is used to tell how 235 many jobs are still available from a job pool. claim all remaining 236 jobs, as we don't want sphinx-build to run in parallel with other 237 jobs. 238 239 Despite that, the user may actually force a different value than 240 the number of available jobs via command line. 241 242 The "with" logic here is used to ensure that the claimed jobs will 243 be freed once subprocess finishes 244 """ 245 246 with JobserverExec() as jobserver: 247 if jobserver.claim: 248 # 249 # when GNU make is used, claim available jobs from jobserver 250 # 251 n_jobs = str(jobserver.claim) 252 else: 253 # 254 # Otherwise, let sphinx decide by default 255 # 256 n_jobs = "auto" 257 258 # 259 # If explicitly requested via command line, override default 260 # 261 if self.n_jobs: 262 n_jobs = str(self.n_jobs) 263 264 cmd = [sys.executable, sphinx_build] 265 cmd += [f"-j{n_jobs}"] 266 cmd += self.sphinxopts 267 cmd += build_args 268 269 if self.verbose: 270 print(" ".join(cmd)) 271 272 return subprocess.call(cmd, *args, **pwargs) 273 274 def handle_html(self, css, output_dir): 275 """ 276 Extra steps for HTML and epub output. 277 278 For such targets, we need to ensure that CSS will be properly 279 copied to the output _static directory 280 """ 281 282 if not css: 283 return 284 285 css = os.path.expanduser(css) 286 if not css.startswith("/"): 287 css = os.path.join(self.srctree, css) 288 289 static_dir = os.path.join(output_dir, "_static") 290 os.makedirs(static_dir, exist_ok=True) 291 292 try: 293 shutil.copy2(css, static_dir) 294 except (OSError, IOError) as e: 295 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 296 297 def build_pdf_file(self, latex_cmd, from_dir, path): 298 """Builds a single pdf file using latex_cmd""" 299 try: 300 subprocess.run(latex_cmd + [path], 301 cwd=from_dir, check=True, env=self.env) 302 303 return True 304 except subprocess.CalledProcessError: 305 return False 306 307 def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): 308 """Build PDF files in parallel if possible""" 309 builds = {} 310 build_failed = False 311 max_len = 0 312 has_tex = False 313 314 # 315 # LaTeX PDF error code is almost useless for us: 316 # any warning makes it non-zero. For kernel doc builds it always return 317 # non-zero even when build succeeds. So, let's do the best next thing: 318 # Ignore build errors. At the end, check if all PDF files were built, 319 # printing a summary with the built ones and returning 0 if all of 320 # them were actually built. 321 # 322 with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: 323 jobs = {} 324 325 for from_dir, pdf_dir, entry in tex_files: 326 name = entry.name 327 328 if not name.endswith(tex_suffix): 329 continue 330 331 name = name[:-len(tex_suffix)] 332 has_tex = True 333 334 future = executor.submit(self.build_pdf_file, latex_cmd, 335 from_dir, entry.path) 336 jobs[future] = (from_dir, pdf_dir, name) 337 338 for future in futures.as_completed(jobs): 339 from_dir, pdf_dir, name = jobs[future] 340 341 pdf_name = name + ".pdf" 342 pdf_from = os.path.join(from_dir, pdf_name) 343 pdf_to = os.path.join(pdf_dir, pdf_name) 344 out_name = os.path.relpath(pdf_to, self.builddir) 345 max_len = max(max_len, len(out_name)) 346 347 try: 348 success = future.result() 349 350 if success and os.path.exists(pdf_from): 351 os.rename(pdf_from, pdf_to) 352 353 # 354 # if verbose, get the name of built PDF file 355 # 356 if self.verbose: 357 builds[out_name] = "SUCCESS" 358 else: 359 builds[out_name] = "FAILED" 360 build_failed = True 361 except futures.Error as e: 362 builds[out_name] = f"FAILED ({repr(e)})" 363 build_failed = True 364 365 # 366 # Handle case where no .tex files were found 367 # 368 if not has_tex: 369 out_name = "LaTeX files" 370 max_len = max(max_len, len(out_name)) 371 builds[out_name] = "FAILED: no .tex files were generated" 372 build_failed = True 373 374 return builds, build_failed, max_len 375 376 def handle_pdf(self, output_dirs, deny_vf): 377 """ 378 Extra steps for PDF output. 379 380 As PDF is handled via a LaTeX output, after building the .tex file, 381 a new build is needed to create the PDF output from the latex 382 directory. 383 """ 384 builds = {} 385 max_len = 0 386 tex_suffix = ".tex" 387 tex_files = [] 388 389 # 390 # Since early 2024, Fedora and openSUSE tumbleweed have started 391 # deploying variable-font format of "Noto CJK", causing LaTeX 392 # to break with CJK. Work around it, by denying the variable font 393 # usage during xelatex build by passing the location of a config 394 # file with a deny list. 395 # 396 # See tools/docs/lib/latex_fonts.py for more details. 397 # 398 if deny_vf: 399 deny_vf = os.path.expanduser(deny_vf) 400 if os.path.isdir(deny_vf): 401 self.env["XDG_CONFIG_HOME"] = deny_vf 402 403 for from_dir in output_dirs: 404 pdf_dir = os.path.join(from_dir, "../pdf") 405 os.makedirs(pdf_dir, exist_ok=True) 406 407 if self.latexmk_cmd: 408 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 409 else: 410 latex_cmd = [self.pdflatex] 411 412 latex_cmd.extend(shlex.split(self.latexopts)) 413 414 # Get a list of tex files to process 415 with os.scandir(from_dir) as it: 416 for entry in it: 417 if entry.name.endswith(tex_suffix): 418 tex_files.append((from_dir, pdf_dir, entry)) 419 420 # 421 # When using make, this won't be used, as the number of jobs comes 422 # from POSIX jobserver. So, this covers the case where build comes 423 # from command line. On such case, serialize by default, except if 424 # the user explicitly sets the number of jobs. 425 # 426 n_jobs = 1 427 428 # n_jobs is either an integer or "auto". Only use it if it is a number 429 if self.n_jobs: 430 try: 431 n_jobs = int(self.n_jobs) 432 except ValueError: 433 pass 434 435 # 436 # When using make, jobserver.claim is the number of jobs that were 437 # used with "-j" and that aren't used by other make targets 438 # 439 with JobserverExec() as jobserver: 440 n_jobs = 1 441 442 # 443 # Handle the case when a parameter is passed via command line, 444 # using it as default, if jobserver doesn't claim anything 445 # 446 if self.n_jobs: 447 try: 448 n_jobs = int(self.n_jobs) 449 except ValueError: 450 pass 451 452 if jobserver.claim: 453 n_jobs = jobserver.claim 454 455 builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, 456 latex_cmd, 457 tex_files, 458 n_jobs) 459 460 # 461 # In verbose mode, print a summary with the build results per file. 462 # Otherwise, print a single line with all failures, if any. 463 # On both cases, return code 1 indicates build failures, 464 # 465 if self.verbose: 466 msg = "Summary" 467 msg += "\n" + "=" * len(msg) 468 print() 469 print(msg) 470 471 for pdf_name, pdf_file in builds.items(): 472 print(f"{pdf_name:<{max_len}}: {pdf_file}") 473 474 print() 475 if build_failed: 476 msg = LatexFontChecker().check() 477 if msg: 478 print(msg) 479 480 sys.exit("Error: not all PDF files were created.") 481 482 elif build_failed: 483 n_failures = len(builds) 484 failures = ", ".join(builds.keys()) 485 486 msg = LatexFontChecker().check() 487 if msg: 488 print(msg) 489 490 sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") 491 492 def handle_info(self, output_dirs): 493 """ 494 Extra steps for Info output. 495 496 For texinfo generation, an additional make is needed from the 497 texinfo directory. 498 """ 499 500 for output_dir in output_dirs: 501 try: 502 subprocess.run(["make", "info"], cwd=output_dir, check=True) 503 except subprocess.CalledProcessError as e: 504 sys.exit(f"Error generating info docs: {e}") 505 506 def cleandocs(self, builder): # pylint: disable=W0613 507 """Remove documentation output directory""" 508 shutil.rmtree(self.builddir, ignore_errors=True) 509 510 def build(self, target, sphinxdirs=None, conf="conf.py", 511 theme=None, css=None, paper=None, deny_vf=None): 512 """ 513 Build documentation using Sphinx. This is the core function of this 514 module. It prepares all arguments required by sphinx-build. 515 """ 516 517 builder = TARGETS[target]["builder"] 518 out_dir = TARGETS[target].get("out_dir", "") 519 520 # 521 # Cleandocs doesn't require sphinx-build 522 # 523 if target == "cleandocs": 524 self.cleandocs(builder) 525 return 526 527 if theme: 528 os.environ["DOCS_THEME"] = theme 529 530 # 531 # Other targets require sphinx-build, so check if it exists 532 # 533 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 534 if not sphinxbuild: 535 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 536 537 if builder == "latex": 538 if not self.pdflatex_cmd and not self.latexmk_cmd: 539 sys.exit("Error: pdflatex or latexmk required for PDF generation") 540 541 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 542 543 # 544 # Fill in base arguments for Sphinx build 545 # 546 kerneldoc = self.kerneldoc 547 if kerneldoc.startswith(self.srctree): 548 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 549 550 args = [ "-b", builder, "-c", docs_dir ] 551 552 if builder == "latex": 553 if not paper: 554 paper = PAPER[1] 555 556 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 557 558 if self.config_rust: 559 args.extend(["-t", "rustdoc"]) 560 561 if conf: 562 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 563 564 if not sphinxdirs: 565 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 566 567 # 568 # The sphinx-build tool has a bug: internally, it tries to set 569 # locale with locale.setlocale(locale.LC_ALL, ''). This causes a 570 # crash if language is not set. Detect and fix it. 571 # 572 try: 573 locale.setlocale(locale.LC_ALL, '') 574 except locale.Error: 575 self.env["LC_ALL"] = "C" 576 577 # 578 # sphinxdirs can be a list or a whitespace-separated string 579 # 580 sphinxdirs_list = [] 581 for sphinxdir in sphinxdirs: 582 if isinstance(sphinxdir, list): 583 sphinxdirs_list += sphinxdir 584 else: 585 sphinxdirs_list += sphinxdir.split() 586 587 # 588 # Step 1: Build each directory in separate. 589 # 590 # This is not the best way of handling it, as cross-references between 591 # them will be broken, but this is what we've been doing since 592 # the beginning. 593 # 594 output_dirs = [] 595 for sphinxdir in sphinxdirs_list: 596 src_dir = os.path.join(docs_dir, sphinxdir) 597 doctree_dir = os.path.join(self.builddir, ".doctrees") 598 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 599 600 # 601 # Make directory names canonical 602 # 603 src_dir = os.path.normpath(src_dir) 604 doctree_dir = os.path.normpath(doctree_dir) 605 output_dir = os.path.normpath(output_dir) 606 607 os.makedirs(doctree_dir, exist_ok=True) 608 os.makedirs(output_dir, exist_ok=True) 609 610 output_dirs.append(output_dir) 611 612 build_args = args + [ 613 "-d", doctree_dir, 614 "-D", f"kerneldoc_bin={kerneldoc}", 615 "-D", f"version={self.kernelversion}", 616 "-D", f"release={self.kernelrelease}", 617 "-D", f"kerneldoc_srctree={self.srctree}", 618 src_dir, 619 output_dir, 620 ] 621 622 try: 623 self.run_sphinx(sphinxbuild, build_args, env=self.env) 624 except (OSError, ValueError, subprocess.SubprocessError) as e: 625 sys.exit(f"Build failed: {repr(e)}") 626 627 # 628 # Ensure that each html/epub output will have needed static files 629 # 630 if target in ["htmldocs", "epubdocs"]: 631 self.handle_html(css, output_dir) 632 633 # 634 # Step 2: Some targets (PDF and info) require an extra step once 635 # sphinx-build finishes 636 # 637 if target == "pdfdocs": 638 self.handle_pdf(output_dirs, deny_vf) 639 elif target == "infodocs": 640 self.handle_info(output_dirs) 641 642def jobs_type(value): 643 """ 644 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 645 equal or bigger than one. 646 """ 647 if value is None: 648 return None 649 650 if value.lower() == 'auto': 651 return value.lower() 652 653 try: 654 if int(value) >= 1: 655 return value 656 657 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 658 except ValueError: 659 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 660 661def main(): 662 """ 663 Main function. The only mandatory argument is the target. If not 664 specified, the other arguments will use default values if not 665 specified at os.environ. 666 """ 667 parser = argparse.ArgumentParser(description="Kernel documentation builder") 668 669 parser.add_argument("target", choices=list(TARGETS.keys()), 670 help="Documentation target to build") 671 parser.add_argument("--sphinxdirs", nargs="+", 672 help="Specific directories to build") 673 parser.add_argument("--conf", default="conf.py", 674 help="Sphinx configuration file") 675 parser.add_argument("--builddir", default="output", 676 help="Sphinx configuration file") 677 678 parser.add_argument("--theme", help="Sphinx theme to use") 679 680 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 681 682 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 683 help="Paper size for LaTeX/PDF output") 684 685 parser.add_argument('--deny-vf', 686 help="Configuration to deny variable fonts on pdf builds") 687 688 parser.add_argument("-v", "--verbose", action='store_true', 689 help="place build in verbose mode") 690 691 parser.add_argument('-j', '--jobs', type=jobs_type, 692 help="Sets number of jobs to use with sphinx-build") 693 694 parser.add_argument('-i', '--interactive', action='store_true', 695 help="Change latex default to run in interactive mode") 696 697 args = parser.parse_args() 698 699 PythonVersion.check_python(MIN_PYTHON_VERSION) 700 701 builder = SphinxBuilder(builddir=args.builddir, 702 verbose=args.verbose, n_jobs=args.jobs, 703 interactive=args.interactive) 704 705 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 706 theme=args.theme, css=args.css, paper=args.paper, 707 deny_vf=args.deny_vf) 708 709if __name__ == "__main__": 710 main() 711