1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# pylint: disable=C0103,C0209 5# 6# 7 8""" 9Interacts with the POSIX jobserver during the Kernel build time. 10 11A "normal" jobserver task, like the one initiated by a make subrocess would do: 12 13 - open read/write file descriptors to communicate with the job server; 14 - ask for one slot by calling:: 15 16 claim = os.read(reader, 1) 17 18 - when the job finshes, call:: 19 20 os.write(writer, b"+") # os.write(writer, claim) 21 22Here, the goal is different: This script aims to get the remaining number 23of slots available, using all of them to run a command which handle tasks in 24parallel. To to that, it has a loop that ends only after there are no 25slots left. It then increments the number by one, in order to allow a 26call equivalent to ``make -j$((claim+1))``, e.g. having a parent make creating 27$claim child to do the actual work. 28 29The end goal here is to keep the total number of build tasks under the 30limit established by the initial ``make -j$n_proc`` call. 31 32See: 33 https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver 34""" 35 36import errno 37import os 38import subprocess 39import sys 40 41class JobserverExec: 42 """ 43 Claim all slots from make using POSIX Jobserver. 44 45 The main methods here are: 46 47 - open(): reserves all slots; 48 - close(): method returns all used slots back to make; 49 - run(): executes a command setting PARALLELISM=<available slots jobs + 1>. 50 """ 51 52 def __init__(self): 53 """Initialize internal vars.""" 54 self.claim = 0 55 self.jobs = b"" 56 self.reader = None 57 self.writer = None 58 self.is_open = False 59 60 def open(self): 61 """Reserve all available slots to be claimed later on.""" 62 63 if self.is_open: 64 return 65 66 try: 67 # Fetch the make environment options. 68 flags = os.environ["MAKEFLAGS"] 69 # Look for "--jobserver=R,W" 70 # Note that GNU Make has used --jobserver-fds and --jobserver-auth 71 # so this handles all of them. 72 opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] 73 74 # Parse out R,W file descriptor numbers and set them nonblocking. 75 # If the MAKEFLAGS variable contains multiple instances of the 76 # --jobserver-auth= option, the last one is relevant. 77 fds = opts[-1].split("=", 1)[1] 78 79 # Starting with GNU Make 4.4, named pipes are used for reader 80 # and writer. 81 # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134 82 _, _, path = fds.partition("fifo:") 83 84 if path: 85 self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) 86 self.writer = os.open(path, os.O_WRONLY) 87 else: 88 self.reader, self.writer = [int(x) for x in fds.split(",", 1)] 89 # Open a private copy of reader to avoid setting nonblocking 90 # on an unexpecting process with the same reader fd. 91 self.reader = os.open("/proc/self/fd/%d" % (self.reader), 92 os.O_RDONLY | os.O_NONBLOCK) 93 94 # Read out as many jobserver slots as possible 95 while True: 96 try: 97 slot = os.read(self.reader, 8) 98 if not slot: 99 # Clear self.jobs to prevent us from probably writing incorrect file. 100 self.jobs = b"" 101 raise ValueError("unexpected empty token from jobserver fd, invalid '--jobserver-auth=' setting?") 102 self.jobs += slot 103 except (OSError, IOError) as e: 104 if e.errno == errno.EWOULDBLOCK: 105 # Stop at the end of the jobserver queue. 106 break 107 # If something went wrong, give back the jobs. 108 if self.jobs: 109 os.write(self.writer, self.jobs) 110 raise e 111 112 # Add a bump for our caller's reserveration, since we're just going 113 # to sit here blocked on our child. 114 self.claim = len(self.jobs) + 1 115 116 except (KeyError, IndexError, ValueError, OSError, IOError) as e: 117 print(f"jobserver: warning: {repr(e)}", file=sys.stderr) 118 # Any missing environment strings or bad fds should result in just 119 # not being parallel. 120 self.claim = None 121 122 self.is_open = True 123 124 def close(self): 125 """Return all reserved slots to Jobserver.""" 126 127 if not self.is_open: 128 return 129 130 # Return all the reserved slots. 131 if len(self.jobs): 132 os.write(self.writer, self.jobs) 133 134 self.is_open = False 135 136 def __enter__(self): 137 self.open() 138 return self 139 140 def __exit__(self, exc_type, exc_value, exc_traceback): 141 self.close() 142 143 def run(self, cmd, *args, **pwargs): 144 """ 145 Run a command setting PARALLELISM env variable to the number of 146 available job slots (claim) + 1, e.g. it will reserve claim slots 147 to do the actual build work, plus one to monitor its children. 148 """ 149 self.open() # Ensure that self.claim is set 150 151 # We can only claim parallelism if there was a jobserver (i.e. a 152 # top-level "-jN" argument) and there were no other failures. Otherwise 153 # leave out the environment variable and let the child figure out what 154 # is best. 155 if self.claim: 156 os.environ["PARALLELISM"] = str(self.claim) 157 158 return subprocess.call(cmd, *args, **pwargs) 159