1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import os 49import shlex 50import shutil 51import subprocess 52import sys 53 54from lib.python_version import PythonVersion 55from lib.latex_fonts import LatexFontChecker 56 57LIB_DIR = "../../scripts/lib" 58SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 59 60sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 61 62from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 63 64# 65# Some constants 66# 67MIN_PYTHON_VERSION = PythonVersion("3.7").version 68PAPER = ["", "a4", "letter"] 69 70TARGETS = { 71 "cleandocs": { "builder": "clean" }, 72 "linkcheckdocs": { "builder": "linkcheck" }, 73 "htmldocs": { "builder": "html" }, 74 "epubdocs": { "builder": "epub", "out_dir": "epub" }, 75 "texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 76 "infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, 77 "latexdocs": { "builder": "latex", "out_dir": "latex" }, 78 "pdfdocs": { "builder": "latex", "out_dir": "latex" }, 79 "xmldocs": { "builder": "xml", "out_dir": "xml" }, 80} 81 82 83# 84# SphinxBuilder class 85# 86 87class SphinxBuilder: 88 """ 89 Handles a sphinx-build target, adding needed arguments to build 90 with the Kernel. 91 """ 92 93 def is_rust_enabled(self): 94 """Check if rust is enabled at .config""" 95 config_path = os.path.join(self.srctree, ".config") 96 if os.path.isfile(config_path): 97 with open(config_path, "r", encoding="utf-8") as f: 98 return "CONFIG_RUST=y" in f.read() 99 return False 100 101 def get_path(self, path, use_cwd=False, abs_path=False): 102 """ 103 Ancillary routine to handle patches the right way, as shell does. 104 105 It first expands "~" and "~user". Then, if patch is not absolute, 106 join self.srctree. Finally, if requested, convert to abspath. 107 """ 108 109 path = os.path.expanduser(path) 110 if not path.startswith("/"): 111 if use_cwd: 112 base = os.getcwd() 113 else: 114 base = self.srctree 115 116 path = os.path.join(base, path) 117 118 if abs_path: 119 return os.path.abspath(path) 120 121 return path 122 123 def get_sphinx_extra_opts(self, n_jobs): 124 """ 125 Get the number of jobs to be used for docs build passed via command 126 line and desired sphinx verbosity. 127 128 The number of jobs can be on different places: 129 130 1) It can be passed via "-j" argument; 131 2) The SPHINXOPTS="-j8" env var may have "-j"; 132 3) if called via GNU make, -j specifies the desired number of jobs. 133 with GNU makefile, this number is available via POSIX jobserver; 134 4) if none of the above is available, it should default to "-jauto", 135 and let sphinx decide the best value. 136 """ 137 138 # 139 # SPHINXOPTS env var, if used, contains extra arguments to be used 140 # by sphinx-build time. Among them, it may contain sphinx verbosity 141 # and desired number of parallel jobs. 142 # 143 parser = argparse.ArgumentParser() 144 parser.add_argument('-j', '--jobs', type=int) 145 parser.add_argument('-q', '--quiet', type=int) 146 147 # 148 # Other sphinx-build arguments go as-is, so place them 149 # at self.sphinxopts, using shell parser 150 # 151 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 152 153 # 154 # Build a list of sphinx args, honoring verbosity here if specified 155 # 156 157 verbose = self.verbose 158 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 159 if sphinx_args.quiet is True: 160 verbose = False 161 162 # 163 # If the user explicitly sets "-j" at command line, use it. 164 # Otherwise, pick it from SPHINXOPTS args 165 # 166 if n_jobs: 167 self.n_jobs = n_jobs 168 elif sphinx_args.jobs: 169 self.n_jobs = sphinx_args.jobs 170 else: 171 self.n_jobs = None 172 173 if not verbose: 174 self.sphinxopts += ["-q"] 175 176 def __init__(self, builddir, verbose=False, n_jobs=None, interactive=None): 177 """Initialize internal variables""" 178 self.verbose = None 179 180 # 181 # Normal variables passed from Kernel's makefile 182 # 183 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 184 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 185 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 186 187 if not interactive: 188 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 189 else: 190 self.latexopts = os.environ.get("LATEXOPTS", "") 191 192 if not verbose: 193 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 194 195 if verbose is not None: 196 self.verbose = verbose 197 198 # 199 # Source tree directory. This needs to be at os.environ, as 200 # Sphinx extensions use it 201 # 202 self.srctree = os.environ.get("srctree") 203 if not self.srctree: 204 self.srctree = "." 205 os.environ["srctree"] = self.srctree 206 207 # 208 # Now that we can expand srctree, get other directories as well 209 # 210 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 211 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 212 "scripts/kernel-doc.py")) 213 self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) 214 215 self.config_rust = self.is_rust_enabled() 216 217 # 218 # Get directory locations for LaTeX build toolchain 219 # 220 self.pdflatex_cmd = shutil.which(self.pdflatex) 221 self.latexmk_cmd = shutil.which("latexmk") 222 223 self.env = os.environ.copy() 224 225 self.get_sphinx_extra_opts(n_jobs) 226 227 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 228 """ 229 Executes sphinx-build using current python3 command. 230 231 When calling via GNU make, POSIX jobserver is used to tell how 232 many jobs are still available from a job pool. claim all remaining 233 jobs, as we don't want sphinx-build to run in parallel with other 234 jobs. 235 236 Despite that, the user may actually force a different value than 237 the number of available jobs via command line. 238 239 The "with" logic here is used to ensure that the claimed jobs will 240 be freed once subprocess finishes 241 """ 242 243 with JobserverExec() as jobserver: 244 if jobserver.claim: 245 # 246 # when GNU make is used, claim available jobs from jobserver 247 # 248 n_jobs = str(jobserver.claim) 249 else: 250 # 251 # Otherwise, let sphinx decide by default 252 # 253 n_jobs = "auto" 254 255 # 256 # If explicitly requested via command line, override default 257 # 258 if self.n_jobs: 259 n_jobs = str(self.n_jobs) 260 261 cmd = [sys.executable, sphinx_build] 262 cmd += [f"-j{n_jobs}"] 263 cmd += self.sphinxopts 264 cmd += build_args 265 266 if self.verbose: 267 print(" ".join(cmd)) 268 269 return subprocess.call(cmd, *args, **pwargs) 270 271 def handle_html(self, css, output_dir): 272 """ 273 Extra steps for HTML and epub output. 274 275 For such targets, we need to ensure that CSS will be properly 276 copied to the output _static directory 277 """ 278 279 if not css: 280 return 281 282 css = os.path.expanduser(css) 283 if not css.startswith("/"): 284 css = os.path.join(self.srctree, css) 285 286 static_dir = os.path.join(output_dir, "_static") 287 os.makedirs(static_dir, exist_ok=True) 288 289 try: 290 shutil.copy2(css, static_dir) 291 except (OSError, IOError) as e: 292 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 293 294 def handle_pdf(self, output_dirs, deny_vf): 295 """ 296 Extra steps for PDF output. 297 298 As PDF is handled via a LaTeX output, after building the .tex file, 299 a new build is needed to create the PDF output from the latex 300 directory. 301 """ 302 builds = {} 303 max_len = 0 304 305 # 306 # Since early 2024, Fedora and openSUSE tumbleweed have started 307 # deploying variable-font format of "Noto CJK", causing LaTeX 308 # to break with CJK. Work around it, by denying the variable font 309 # usage during xelatex build by passing the location of a config 310 # file with a deny list. 311 # 312 # See tools/docs/lib/latex_fonts.py for more details. 313 # 314 if deny_vf: 315 deny_vf = os.path.expanduser(deny_vf) 316 if os.path.isdir(deny_vf): 317 self.env["XDG_CONFIG_HOME"] = deny_vf 318 319 for from_dir in output_dirs: 320 pdf_dir = os.path.join(from_dir, "../pdf") 321 os.makedirs(pdf_dir, exist_ok=True) 322 323 if self.latexmk_cmd: 324 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 325 else: 326 latex_cmd = [self.pdflatex] 327 328 latex_cmd.extend(shlex.split(self.latexopts)) 329 330 tex_suffix = ".tex" 331 332 # 333 # Process each .tex file 334 # 335 336 has_tex = False 337 build_failed = False 338 with os.scandir(from_dir) as it: 339 for entry in it: 340 if not entry.name.endswith(tex_suffix): 341 continue 342 343 name = entry.name[:-len(tex_suffix)] 344 has_tex = True 345 346 # 347 # LaTeX PDF error code is almost useless for us: 348 # any warning makes it non-zero. For kernel doc builds it 349 # always return non-zero even when build succeeds. 350 # So, let's do the best next thing: check if all PDF 351 # files were built. If they're, print a summary and 352 # return 0 at the end of this function 353 # 354 try: 355 subprocess.run(latex_cmd + [entry.path], 356 cwd=from_dir, check=True, env=self.env) 357 except subprocess.CalledProcessError: 358 pass 359 360 pdf_name = name + ".pdf" 361 pdf_from = os.path.join(from_dir, pdf_name) 362 pdf_to = os.path.join(pdf_dir, pdf_name) 363 364 if os.path.exists(pdf_from): 365 os.rename(pdf_from, pdf_to) 366 builds[name] = os.path.relpath(pdf_to, self.builddir) 367 else: 368 builds[name] = "FAILED" 369 build_failed = True 370 371 name = entry.name.removesuffix(".tex") 372 max_len = max(max_len, len(name)) 373 374 if not has_tex: 375 name = os.path.basename(from_dir) 376 max_len = max(max_len, len(name)) 377 builds[name] = "FAILED (no .tex)" 378 build_failed = True 379 380 msg = "Summary" 381 msg += "\n" + "=" * len(msg) 382 print() 383 print(msg) 384 385 for pdf_name, pdf_file in builds.items(): 386 print(f"{pdf_name:<{max_len}}: {pdf_file}") 387 388 print() 389 390 if build_failed: 391 msg = LatexFontChecker().check() 392 if msg: 393 print(msg) 394 395 sys.exit("PDF build failed: not all PDF files were created.") 396 else: 397 print("All PDF files were built.") 398 399 def handle_info(self, output_dirs): 400 """ 401 Extra steps for Info output. 402 403 For texinfo generation, an additional make is needed from the 404 texinfo directory. 405 """ 406 407 for output_dir in output_dirs: 408 try: 409 subprocess.run(["make", "info"], cwd=output_dir, check=True) 410 except subprocess.CalledProcessError as e: 411 sys.exit(f"Error generating info docs: {e}") 412 413 def cleandocs(self, builder): # pylint: disable=W0613 414 """Remove documentation output directory""" 415 shutil.rmtree(self.builddir, ignore_errors=True) 416 417 def build(self, target, sphinxdirs=None, conf="conf.py", 418 theme=None, css=None, paper=None, deny_vf=None): 419 """ 420 Build documentation using Sphinx. This is the core function of this 421 module. It prepares all arguments required by sphinx-build. 422 """ 423 424 builder = TARGETS[target]["builder"] 425 out_dir = TARGETS[target].get("out_dir", "") 426 427 # 428 # Cleandocs doesn't require sphinx-build 429 # 430 if target == "cleandocs": 431 self.cleandocs(builder) 432 return 433 434 if theme: 435 os.environ["DOCS_THEME"] = theme 436 437 # 438 # Other targets require sphinx-build, so check if it exists 439 # 440 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 441 if not sphinxbuild: 442 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 443 444 if builder == "latex": 445 if not self.pdflatex_cmd and not self.latexmk_cmd: 446 sys.exit("Error: pdflatex or latexmk required for PDF generation") 447 448 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 449 450 # 451 # Fill in base arguments for Sphinx build 452 # 453 kerneldoc = self.kerneldoc 454 if kerneldoc.startswith(self.srctree): 455 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 456 457 args = [ "-b", builder, "-c", docs_dir ] 458 459 if builder == "latex": 460 if not paper: 461 paper = PAPER[1] 462 463 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 464 465 if self.config_rust: 466 args.extend(["-t", "rustdoc"]) 467 468 if conf: 469 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 470 471 if not sphinxdirs: 472 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 473 474 # 475 # sphinxdirs can be a list or a whitespace-separated string 476 # 477 sphinxdirs_list = [] 478 for sphinxdir in sphinxdirs: 479 if isinstance(sphinxdir, list): 480 sphinxdirs_list += sphinxdir 481 else: 482 sphinxdirs_list += sphinxdir.split() 483 484 # 485 # Step 1: Build each directory in separate. 486 # 487 # This is not the best way of handling it, as cross-references between 488 # them will be broken, but this is what we've been doing since 489 # the beginning. 490 # 491 output_dirs = [] 492 for sphinxdir in sphinxdirs_list: 493 src_dir = os.path.join(docs_dir, sphinxdir) 494 doctree_dir = os.path.join(self.builddir, ".doctrees") 495 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 496 497 # 498 # Make directory names canonical 499 # 500 src_dir = os.path.normpath(src_dir) 501 doctree_dir = os.path.normpath(doctree_dir) 502 output_dir = os.path.normpath(output_dir) 503 504 os.makedirs(doctree_dir, exist_ok=True) 505 os.makedirs(output_dir, exist_ok=True) 506 507 output_dirs.append(output_dir) 508 509 build_args = args + [ 510 "-d", doctree_dir, 511 "-D", f"kerneldoc_bin={kerneldoc}", 512 "-D", f"version={self.kernelversion}", 513 "-D", f"release={self.kernelrelease}", 514 "-D", f"kerneldoc_srctree={self.srctree}", 515 src_dir, 516 output_dir, 517 ] 518 519 try: 520 self.run_sphinx(sphinxbuild, build_args, env=self.env) 521 except (OSError, ValueError, subprocess.SubprocessError) as e: 522 sys.exit(f"Build failed: {repr(e)}") 523 524 # 525 # Ensure that each html/epub output will have needed static files 526 # 527 if target in ["htmldocs", "epubdocs"]: 528 self.handle_html(css, output_dir) 529 530 # 531 # Step 2: Some targets (PDF and info) require an extra step once 532 # sphinx-build finishes 533 # 534 if target == "pdfdocs": 535 self.handle_pdf(output_dirs, deny_vf) 536 elif target == "infodocs": 537 self.handle_info(output_dirs) 538 539def jobs_type(value): 540 """ 541 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 542 equal or bigger than one. 543 """ 544 if value is None: 545 return None 546 547 if value.lower() == 'auto': 548 return value.lower() 549 550 try: 551 if int(value) >= 1: 552 return value 553 554 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 555 except ValueError: 556 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 557 558def main(): 559 """ 560 Main function. The only mandatory argument is the target. If not 561 specified, the other arguments will use default values if not 562 specified at os.environ. 563 """ 564 parser = argparse.ArgumentParser(description="Kernel documentation builder") 565 566 parser.add_argument("target", choices=list(TARGETS.keys()), 567 help="Documentation target to build") 568 parser.add_argument("--sphinxdirs", nargs="+", 569 help="Specific directories to build") 570 parser.add_argument("--conf", default="conf.py", 571 help="Sphinx configuration file") 572 parser.add_argument("--builddir", default="output", 573 help="Sphinx configuration file") 574 575 parser.add_argument("--theme", help="Sphinx theme to use") 576 577 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 578 579 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 580 help="Paper size for LaTeX/PDF output") 581 582 parser.add_argument('--deny-vf', 583 help="Configuration to deny variable fonts on pdf builds") 584 585 parser.add_argument("-v", "--verbose", action='store_true', 586 help="place build in verbose mode") 587 588 parser.add_argument('-j', '--jobs', type=jobs_type, 589 help="Sets number of jobs to use with sphinx-build") 590 591 parser.add_argument('-i', '--interactive', action='store_true', 592 help="Change latex default to run in interactive mode") 593 594 args = parser.parse_args() 595 596 PythonVersion.check_python(MIN_PYTHON_VERSION) 597 598 builder = SphinxBuilder(builddir=args.builddir, 599 verbose=args.verbose, n_jobs=args.jobs, 600 interactive=args.interactive) 601 602 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 603 theme=args.theme, css=args.css, paper=args.paper, 604 deny_vf=args.deny_vf) 605 606if __name__ == "__main__": 607 main() 608