1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# Copyright (C) 2025 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 4# 5# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 6# 7# Converted from docs Makefile and parallel-wrapper.sh, both under 8# GPLv2, copyrighted since 2008 by the following authors: 9# 10# Akira Yokosawa <akiyks@gmail.com> 11# Arnd Bergmann <arnd@arndb.de> 12# Breno Leitao <leitao@debian.org> 13# Carlos Bilbao <carlos.bilbao@amd.com> 14# Dave Young <dyoung@redhat.com> 15# Donald Hunter <donald.hunter@gmail.com> 16# Geert Uytterhoeven <geert+renesas@glider.be> 17# Jani Nikula <jani.nikula@intel.com> 18# Jan Stancek <jstancek@redhat.com> 19# Jonathan Corbet <corbet@lwn.net> 20# Joshua Clayton <stillcompiling@gmail.com> 21# Kees Cook <keescook@chromium.org> 22# Linus Torvalds <torvalds@linux-foundation.org> 23# Magnus Damm <damm+renesas@opensource.se> 24# Masahiro Yamada <masahiroy@kernel.org> 25# Mauro Carvalho Chehab <mchehab+huawei@kernel.org> 26# Maxim Cournoyer <maxim.cournoyer@gmail.com> 27# Peter Foley <pefoley2@pefoley.com> 28# Randy Dunlap <rdunlap@infradead.org> 29# Rob Herring <robh@kernel.org> 30# Shuah Khan <shuahkh@osg.samsung.com> 31# Thorsten Blum <thorsten.blum@toblux.com> 32# Tomas Winkler <tomas.winkler@intel.com> 33 34 35""" 36Sphinx build wrapper that handles Kernel-specific business rules: 37 38- it gets the Kernel build environment vars; 39- it determines what's the best parallelism; 40- it handles SPHINXDIRS 41 42This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is 43below that, it seeks for a new Python version. If found, it re-runs using 44the newer version. 45""" 46 47import argparse 48import locale 49import os 50import re 51import shlex 52import shutil 53import subprocess 54import sys 55 56from concurrent import futures 57from glob import glob 58 59LIB_DIR = "lib" 60SRC_DIR = os.path.dirname(os.path.realpath(__file__)) 61 62sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) 63 64from jobserver import JobserverExec # pylint: disable=C0413 65 66 67def parse_version(version): 68 """Convert a major.minor.patch version into a tuple""" 69 return tuple(int(x) for x in version.split(".")) 70 71def ver_str(version): 72 """Returns a version tuple as major.minor.patch""" 73 74 return ".".join([str(x) for x in version]) 75 76# Minimal supported Python version needed by Sphinx and its extensions 77MIN_PYTHON_VERSION = parse_version("3.7") 78 79# Default value for --venv parameter 80VENV_DEFAULT = "sphinx_latest" 81 82# List of make targets and its corresponding builder and output directory 83TARGETS = { 84 "cleandocs": { 85 "builder": "clean", 86 }, 87 "htmldocs": { 88 "builder": "html", 89 }, 90 "epubdocs": { 91 "builder": "epub", 92 "out_dir": "epub", 93 }, 94 "texinfodocs": { 95 "builder": "texinfo", 96 "out_dir": "texinfo", 97 }, 98 "infodocs": { 99 "builder": "texinfo", 100 "out_dir": "texinfo", 101 }, 102 "latexdocs": { 103 "builder": "latex", 104 "out_dir": "latex", 105 }, 106 "pdfdocs": { 107 "builder": "latex", 108 "out_dir": "latex", 109 }, 110 "xmldocs": { 111 "builder": "xml", 112 "out_dir": "xml", 113 }, 114 "linkcheckdocs": { 115 "builder": "linkcheck" 116 }, 117} 118 119# Paper sizes. An empty value will pick the default 120PAPER = ["", "a4", "letter"] 121 122class SphinxBuilder: 123 """ 124 Handles a sphinx-build target, adding needed arguments to build 125 with the Kernel. 126 """ 127 128 def is_rust_enabled(self): 129 """Check if rust is enabled at .config""" 130 config_path = os.path.join(self.srctree, ".config") 131 if os.path.isfile(config_path): 132 with open(config_path, "r", encoding="utf-8") as f: 133 return "CONFIG_RUST=y" in f.read() 134 return False 135 136 def get_path(self, path, abs_path=False): 137 """ 138 Ancillary routine to handle patches the right way, as shell does. 139 140 It first expands "~" and "~user". Then, if patch is not absolute, 141 join self.srctree. Finally, if requested, convert to abspath. 142 """ 143 144 path = os.path.expanduser(path) 145 if not path.startswith("/"): 146 path = os.path.join(self.srctree, path) 147 148 if abs_path: 149 return os.path.abspath(path) 150 151 return path 152 153 def __init__(self, venv=None, verbose=False, n_jobs=None, interactive=None): 154 """Initialize internal variables""" 155 self.venv = venv 156 self.verbose = None 157 158 # Normal variables passed from Kernel's makefile 159 self.kernelversion = os.environ.get("KERNELVERSION", "unknown") 160 self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") 161 self.pdflatex = os.environ.get("PDFLATEX", "xelatex") 162 163 if not interactive: 164 self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") 165 else: 166 self.latexopts = os.environ.get("LATEXOPTS", "") 167 168 if not verbose: 169 verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") 170 171 # Handle SPHINXOPTS evironment 172 sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) 173 174 # As we handle number of jobs and quiet in separate, we need to pick 175 # it the same way as sphinx-build would pick, so let's use argparse 176 # do to the right argument expansion 177 parser = argparse.ArgumentParser() 178 parser.add_argument('-j', '--jobs', type=int) 179 parser.add_argument('-q', '--quiet', type=int) 180 181 # Other sphinx-build arguments go as-is, so place them 182 # at self.sphinxopts 183 sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) 184 if sphinx_args.quiet == True: 185 self.verbose = False 186 187 if sphinx_args.jobs: 188 self.n_jobs = sphinx_args.jobs 189 190 # Command line arguments was passed, override SPHINXOPTS 191 if verbose is not None: 192 self.verbose = verbose 193 194 self.n_jobs = n_jobs 195 196 # Source tree directory. This needs to be at os.environ, as 197 # Sphinx extensions and media uAPI makefile needs it 198 self.srctree = os.environ.get("srctree") 199 if not self.srctree: 200 self.srctree = "." 201 os.environ["srctree"] = self.srctree 202 203 # Now that we can expand srctree, get other directories as well 204 self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") 205 self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", 206 "scripts/kernel-doc.py")) 207 self.obj = os.environ.get("obj", "Documentation") 208 self.builddir = self.get_path(os.path.join(self.obj, "output"), 209 abs_path=True) 210 211 # Media uAPI needs it 212 os.environ["BUILDDIR"] = self.builddir 213 214 # Detect if rust is enabled 215 self.config_rust = self.is_rust_enabled() 216 217 # Get directory locations for LaTeX build toolchain 218 self.pdflatex_cmd = shutil.which(self.pdflatex) 219 self.latexmk_cmd = shutil.which("latexmk") 220 221 self.env = os.environ.copy() 222 223 # If venv parameter is specified, run Sphinx from venv 224 if venv: 225 bin_dir = os.path.join(venv, "bin") 226 if os.path.isfile(os.path.join(bin_dir, "activate")): 227 # "activate" virtual env 228 self.env["PATH"] = bin_dir + ":" + self.env["PATH"] 229 self.env["VIRTUAL_ENV"] = venv 230 if "PYTHONHOME" in self.env: 231 del self.env["PYTHONHOME"] 232 print(f"Setting venv to {venv}") 233 else: 234 sys.exit(f"Venv {venv} not found.") 235 236 def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): 237 """ 238 Executes sphinx-build using current python3 command and setting 239 -j parameter if possible to run the build in parallel. 240 """ 241 242 with JobserverExec() as jobserver: 243 if jobserver.claim: 244 n_jobs = str(jobserver.claim) 245 else: 246 n_jobs = "auto" # Supported since Sphinx 1.7 247 248 cmd = [] 249 250 if self.venv: 251 cmd.append("python") 252 else: 253 cmd.append(sys.executable) 254 255 cmd.append(sphinx_build) 256 257 # if present, SPHINXOPTS or command line --jobs overrides default 258 if self.n_jobs: 259 n_jobs = str(self.n_jobs) 260 261 if n_jobs: 262 cmd += [f"-j{n_jobs}"] 263 264 if not self.verbose: 265 cmd.append("-q") 266 267 cmd += self.sphinxopts 268 269 cmd += build_args 270 271 if self.verbose: 272 print(" ".join(cmd)) 273 274 rc = subprocess.call(cmd, *args, **pwargs) 275 276 def handle_html(self, css, output_dir): 277 """ 278 Extra steps for HTML and epub output. 279 280 For such targets, we need to ensure that CSS will be properly 281 copied to the output _static directory 282 """ 283 284 if not css: 285 return 286 287 css = os.path.expanduser(css) 288 if not css.startswith("/"): 289 css = os.path.join(self.srctree, css) 290 291 static_dir = os.path.join(output_dir, "_static") 292 os.makedirs(static_dir, exist_ok=True) 293 294 try: 295 shutil.copy2(css, static_dir) 296 except (OSError, IOError) as e: 297 print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) 298 299 def build_pdf_file(self, latex_cmd, from_dir, path): 300 """Builds a single pdf file using latex_cmd""" 301 try: 302 subprocess.run(latex_cmd + [path], 303 cwd=from_dir, check=True) 304 305 return True 306 except subprocess.CalledProcessError: 307 # LaTeX PDF error code is almost useless: it returns 308 # error codes even when build succeeds but has warnings. 309 # So, we'll ignore the results 310 return False 311 312 def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): 313 """Build PDF files in parallel if possible""" 314 builds = {} 315 build_failed = False 316 max_len = 0 317 has_tex = False 318 319 # Process files in parallel 320 with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: 321 jobs = {} 322 323 for from_dir, pdf_dir, entry in tex_files: 324 name = entry.name 325 326 if not name.endswith(tex_suffix): 327 continue 328 329 name = name[:-len(tex_suffix)] 330 331 max_len = max(max_len, len(name)) 332 333 has_tex = True 334 335 future = executor.submit(self.build_pdf_file, latex_cmd, 336 from_dir, entry.path) 337 jobs[future] = (from_dir, name, entry.path) 338 339 for future in futures.as_completed(jobs): 340 from_dir, name, path = jobs[future] 341 342 pdf_name = name + ".pdf" 343 pdf_from = os.path.join(from_dir, pdf_name) 344 345 try: 346 success = future.result() 347 348 if success and os.path.exists(pdf_from): 349 pdf_to = os.path.join(pdf_dir, pdf_name) 350 351 os.rename(pdf_from, pdf_to) 352 builds[name] = os.path.relpath(pdf_to, self.builddir) 353 else: 354 builds[name] = "FAILED" 355 build_failed = True 356 except Exception as e: 357 builds[name] = f"FAILED ({str(e)})" 358 build_failed = True 359 360 # Handle case where no .tex files were found 361 if not has_tex: 362 name = "Sphinx LaTeX builder" 363 max_len = max(max_len, len(name)) 364 builds[name] = "FAILED (no .tex file was generated)" 365 build_failed = True 366 367 return builds, build_failed, max_len 368 369 def handle_pdf(self, output_dirs): 370 """ 371 Extra steps for PDF output. 372 373 As PDF is handled via a LaTeX output, after building the .tex file, 374 a new build is needed to create the PDF output from the latex 375 directory. 376 """ 377 builds = {} 378 max_len = 0 379 tex_suffix = ".tex" 380 381 # Get all tex files that will be used for PDF build 382 tex_files = [] 383 for from_dir in output_dirs: 384 pdf_dir = os.path.join(from_dir, "../pdf") 385 os.makedirs(pdf_dir, exist_ok=True) 386 387 if self.latexmk_cmd: 388 latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] 389 else: 390 latex_cmd = [self.pdflatex] 391 392 latex_cmd.extend(shlex.split(self.latexopts)) 393 394 # Get a list of tex files to process 395 with os.scandir(from_dir) as it: 396 for entry in it: 397 if entry.name.endswith(tex_suffix): 398 tex_files.append((from_dir, pdf_dir, entry)) 399 400 # When using make, this won't be used, as the number of jobs comes 401 # from POSIX jobserver. So, this covers the case where build comes 402 # from command line. On such case, serialize by default, except if 403 # the user explicitly sets the number of jobs. 404 n_jobs = 1 405 406 # n_jobs is either an integer or "auto". Only use it if it is a number 407 if self.n_jobs: 408 try: 409 n_jobs = int(self.n_jobs) 410 except ValueError: 411 pass 412 413 # When using make, jobserver.claim is the number of jobs that were 414 # used with "-j" and that aren't used by other make targets 415 with JobserverExec() as jobserver: 416 n_jobs = 1 417 418 # Handle the case when a parameter is passed via command line, 419 # using it as default, if jobserver doesn't claim anything 420 if self.n_jobs: 421 try: 422 n_jobs = int(self.n_jobs) 423 except ValueError: 424 pass 425 426 if jobserver.claim: 427 n_jobs = jobserver.claim 428 429 # Build files in parallel 430 builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, 431 latex_cmd, 432 tex_files, 433 n_jobs) 434 435 msg = "Summary" 436 msg += "\n" + "=" * len(msg) 437 print() 438 print(msg) 439 440 for pdf_name, pdf_file in builds.items(): 441 print(f"{pdf_name:<{max_len}}: {pdf_file}") 442 443 print() 444 445 # return an error if a PDF file is missing 446 447 if build_failed: 448 sys.exit(f"PDF build failed: not all PDF files were created.") 449 else: 450 print("All PDF files were built.") 451 452 def handle_info(self, output_dirs): 453 """ 454 Extra steps for Info output. 455 456 For texinfo generation, an additional make is needed from the 457 texinfo directory. 458 """ 459 460 for output_dir in output_dirs: 461 try: 462 subprocess.run(["make", "info"], cwd=output_dir, check=True) 463 except subprocess.CalledProcessError as e: 464 sys.exit(f"Error generating info docs: {e}") 465 466 def cleandocs(self, builder): 467 468 shutil.rmtree(self.builddir, ignore_errors=True) 469 470 def build(self, target, sphinxdirs=None, conf="conf.py", 471 theme=None, css=None, paper=None): 472 """ 473 Build documentation using Sphinx. This is the core function of this 474 module. It prepares all arguments required by sphinx-build. 475 """ 476 477 builder = TARGETS[target]["builder"] 478 out_dir = TARGETS[target].get("out_dir", "") 479 480 # Cleandocs doesn't require sphinx-build 481 if target == "cleandocs": 482 self.cleandocs(builder) 483 return 484 485 # Other targets require sphinx-build 486 sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) 487 if not sphinxbuild: 488 sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") 489 490 if builder == "latex": 491 if not self.pdflatex_cmd and not self.latexmk_cmd: 492 sys.exit("Error: pdflatex or latexmk required for PDF generation") 493 494 docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) 495 496 # Prepare base arguments for Sphinx build 497 kerneldoc = self.kerneldoc 498 if kerneldoc.startswith(self.srctree): 499 kerneldoc = os.path.relpath(kerneldoc, self.srctree) 500 501 # Prepare common Sphinx options 502 args = [ 503 "-b", builder, 504 "-c", docs_dir, 505 ] 506 507 if builder == "latex": 508 if not paper: 509 paper = PAPER[1] 510 511 args.extend(["-D", f"latex_elements.papersize={paper}paper"]) 512 513 if self.config_rust: 514 args.extend(["-t", "rustdoc"]) 515 516 if conf: 517 self.env["SPHINX_CONF"] = self.get_path(conf, abs_path=True) 518 519 if not sphinxdirs: 520 sphinxdirs = os.environ.get("SPHINXDIRS", ".") 521 522 # The sphinx-build tool has a bug: internally, it tries to set 523 # locale with locale.setlocale(locale.LC_ALL, ''). This causes a 524 # crash if language is not set. Detect and fix it. 525 try: 526 locale.setlocale(locale.LC_ALL, '') 527 except Exception: 528 self.env["LC_ALL"] = "C" 529 self.env["LANG"] = "C" 530 531 # sphinxdirs can be a list or a whitespace-separated string 532 sphinxdirs_list = [] 533 for sphinxdir in sphinxdirs: 534 if isinstance(sphinxdir, list): 535 sphinxdirs_list += sphinxdir 536 else: 537 for name in sphinxdir.split(" "): 538 sphinxdirs_list.append(name) 539 540 # Build each directory 541 output_dirs = [] 542 for sphinxdir in sphinxdirs_list: 543 src_dir = os.path.join(docs_dir, sphinxdir) 544 doctree_dir = os.path.join(self.builddir, ".doctrees") 545 output_dir = os.path.join(self.builddir, sphinxdir, out_dir) 546 547 # Make directory names canonical 548 src_dir = os.path.normpath(src_dir) 549 doctree_dir = os.path.normpath(doctree_dir) 550 output_dir = os.path.normpath(output_dir) 551 552 os.makedirs(doctree_dir, exist_ok=True) 553 os.makedirs(output_dir, exist_ok=True) 554 555 output_dirs.append(output_dir) 556 557 build_args = args + [ 558 "-d", doctree_dir, 559 "-D", f"kerneldoc_bin={kerneldoc}", 560 "-D", f"version={self.kernelversion}", 561 "-D", f"release={self.kernelrelease}", 562 "-D", f"kerneldoc_srctree={self.srctree}", 563 src_dir, 564 output_dir, 565 ] 566 567 # Execute sphinx-build 568 try: 569 self.run_sphinx(sphinxbuild, build_args, env=self.env) 570 except Exception as e: 571 sys.exit(f"Build failed: {e}") 572 573 # Ensure that html/epub will have needed static files 574 if target in ["htmldocs", "epubdocs"]: 575 self.handle_html(css, output_dir) 576 577 # PDF and Info require a second build step 578 if target == "pdfdocs": 579 self.handle_pdf(output_dirs) 580 elif target == "infodocs": 581 self.handle_info(output_dirs) 582 583 @staticmethod 584 def get_python_version(cmd): 585 """ 586 Get python version from a Python binary. As we need to detect if 587 are out there newer python binaries, we can't rely on sys.release here. 588 """ 589 590 result = subprocess.run([cmd, "--version"], check=True, 591 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 592 universal_newlines=True) 593 version = result.stdout.strip() 594 595 match = re.search(r"(\d+\.\d+\.\d+)", version) 596 if match: 597 return parse_version(match.group(1)) 598 599 print(f"Can't parse version {version}") 600 return (0, 0, 0) 601 602 @staticmethod 603 def find_python(): 604 """ 605 Detect if are out there any python 3.xy version newer than the 606 current one. 607 608 Note: this routine is limited to up to 2 digits for python3. We 609 may need to update it one day, hopefully on a distant future. 610 """ 611 patterns = [ 612 "python3.[0-9]", 613 "python3.[0-9][0-9]", 614 ] 615 616 # Seek for a python binary newer than MIN_PYTHON_VERSION 617 for path in os.getenv("PATH", "").split(":"): 618 for pattern in patterns: 619 for cmd in glob(os.path.join(path, pattern)): 620 if os.path.isfile(cmd) and os.access(cmd, os.X_OK): 621 version = SphinxBuilder.get_python_version(cmd) 622 if version >= MIN_PYTHON_VERSION: 623 return cmd 624 625 return None 626 627 @staticmethod 628 def check_python(): 629 """ 630 Check if the current python binary satisfies our minimal requirement 631 for Sphinx build. If not, re-run with a newer version if found. 632 """ 633 cur_ver = sys.version_info[:3] 634 if cur_ver >= MIN_PYTHON_VERSION: 635 return 636 637 python_ver = ver_str(cur_ver) 638 639 new_python_cmd = SphinxBuilder.find_python() 640 if not new_python_cmd: 641 sys.exit(f"Python version {python_ver} is not supported anymore.") 642 643 # Restart script using the newer version 644 script_path = os.path.abspath(sys.argv[0]) 645 args = [new_python_cmd, script_path] + sys.argv[1:] 646 647 print(f"Python {python_ver} not supported. Changing to {new_python_cmd}") 648 649 try: 650 os.execv(new_python_cmd, args) 651 except OSError as e: 652 sys.exit(f"Failed to restart with {new_python_cmd}: {e}") 653 654def jobs_type(value): 655 """ 656 Handle valid values for -j. Accepts Sphinx "-jauto", plus a number 657 equal or bigger than one. 658 """ 659 if value is None: 660 return None 661 662 if value.lower() == 'auto': 663 return value.lower() 664 665 try: 666 if int(value) >= 1: 667 return value 668 669 raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") 670 except ValueError: 671 raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") 672 673def main(): 674 """ 675 Main function. The only mandatory argument is the target. If not 676 specified, the other arguments will use default values if not 677 specified at os.environ. 678 """ 679 parser = argparse.ArgumentParser(description="Kernel documentation builder") 680 681 parser.add_argument("target", choices=list(TARGETS.keys()), 682 help="Documentation target to build") 683 parser.add_argument("--sphinxdirs", nargs="+", 684 help="Specific directories to build") 685 parser.add_argument("--conf", default="conf.py", 686 help="Sphinx configuration file") 687 688 parser.add_argument("--theme", help="Sphinx theme to use") 689 690 parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") 691 692 parser.add_argument("--paper", choices=PAPER, default=PAPER[0], 693 help="Paper size for LaTeX/PDF output") 694 695 parser.add_argument("-v", "--verbose", action='store_true', 696 help="place build in verbose mode") 697 698 parser.add_argument('-j', '--jobs', type=jobs_type, 699 help="Sets number of jobs to use with sphinx-build") 700 701 parser.add_argument('-i', '--interactive', action='store_true', 702 help="Change latex default to run in interactive mode") 703 704 parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', 705 default=None, 706 help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') 707 708 args = parser.parse_args() 709 710 SphinxBuilder.check_python() 711 712 builder = SphinxBuilder(venv=args.venv, verbose=args.verbose, 713 n_jobs=args.jobs, interactive=args.interactive) 714 715 builder.build(args.target, sphinxdirs=args.sphinxdirs, conf=args.conf, 716 theme=args.theme, css=args.css, paper=args.paper) 717 718if __name__ == "__main__": 719 main() 720