1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# pylint: disable=C0103,C0209 5# 6# 7 8""" 9Interacts with the POSIX jobserver during the Kernel build time. 10 11A "normal" jobserver task, like the one initiated by a make subprocess would do: 12 13 - open read/write file descriptors to communicate with the job server; 14 - ask for one slot by calling:: 15 16 claim = os.read(reader, 1) 17 18 - when the job finishes, call:: 19 20 os.write(writer, b"+") # os.write(writer, claim) 21 22Here, the goal is different: This script aims to get the remaining number 23of slots available, using all of them to run a command which handle tasks in 24parallel. To to that, it has a loop that ends only after there are no 25slots left. It then increments the number by one, in order to allow a 26call equivalent to ``make -j$((claim+1))``, e.g. having a parent make creating 27$claim child to do the actual work. 28 29The end goal here is to keep the total number of build tasks under the 30limit established by the initial ``make -j$n_proc`` call. 31 32A command that takes its thread count on the command line rather than from 33the environment can use the ``%PARALLELISM%`` token in its arguments, which 34is replaced by the count, or by 1 when there is no jobserver. 35 36See: 37 https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver 38""" 39 40import errno 41import os 42import subprocess 43import sys 44 45PARALLELISM_TOKEN = "%PARALLELISM%" 46 47def warn(text, *args): 48 print(f'WARNING: {text}', *args, file = sys.stderr) 49 50class JobserverExec: 51 """ 52 Claim all slots from make using POSIX Jobserver. 53 54 The main methods here are: 55 56 - open(): reserves all slots; 57 - close(): method returns all used slots back to make; 58 - run(): executes a command setting PARALLELISM=<available slots jobs + 1>. 59 """ 60 61 def __init__(self): 62 """Initialize internal vars.""" 63 self.claim = 0 64 self.jobs = b"" 65 self.reader = None 66 self.writer = None 67 self.is_open = False 68 69 def open(self): 70 """Reserve all available slots to be claimed later on.""" 71 72 if self.is_open: 73 return 74 self.is_open = True # We only try once 75 self.claim = None 76 # 77 # Check the make flags for "--jobserver=R,W" 78 # Note that GNU Make has used --jobserver-fds and --jobserver-auth 79 # so this handles all of them. 80 # 81 flags = os.environ.get('MAKEFLAGS', '') 82 opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] 83 if not opts: 84 return 85 # 86 # Separate out the provided file descriptors 87 # 88 split_opt = opts[-1].split('=', 1) 89 if len(split_opt) != 2: 90 warn('unparseable option:', opts[-1]) 91 return 92 fds = split_opt[1] 93 # 94 # As of GNU Make 4.4, we'll be looking for a named pipe 95 # identified as fifo:path 96 # 97 if fds.startswith('fifo:'): 98 path = fds[len('fifo:'):] 99 try: 100 self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) 101 self.writer = os.open(path, os.O_WRONLY) 102 except (OSError, IOError): 103 warn('unable to open jobserver pipe', path) 104 return 105 # 106 # Otherwise look for integer file-descriptor numbers. 107 # 108 else: 109 split_fds = fds.split(',') 110 if len(split_fds) != 2: 111 warn('malformed jobserver file descriptors:', fds) 112 return 113 try: 114 self.reader = int(split_fds[0]) 115 self.writer = int(split_fds[1]) 116 except ValueError: 117 warn('non-integer jobserver file-descriptors:', fds) 118 return 119 try: 120 # 121 # Open a private copy of reader to avoid setting nonblocking 122 # on an unexpecting process with the same reader fd. 123 # 124 self.reader = os.open(f"/proc/self/fd/{self.reader}", 125 os.O_RDONLY | os.O_NONBLOCK) 126 except (IOError, OSError) as e: 127 warn('Unable to reopen jobserver read-side pipe:', repr(e)) 128 return 129 # 130 # OK, we have the channel to the job server; read out as many jobserver 131 # slots as possible. 132 # 133 while True: 134 try: 135 slot = os.read(self.reader, 8) 136 if not slot: 137 # 138 # Something went wrong. Clear self.jobs to avoid writing 139 # weirdness back to the jobserver and give up. 140 self.jobs = b"" 141 warn("unexpected empty token from jobserver;" 142 " possible invalid '--jobserver-auth=' setting") 143 self.claim = None 144 return 145 except (OSError, IOError) as e: 146 # 147 # If there is nothing more to read then we are done. 148 # 149 if e.errno == errno.EWOULDBLOCK: 150 break 151 # 152 # Anything else says that something went weird; give back 153 # the jobs and give up. 154 # 155 if self.jobs: 156 os.write(self.writer, self.jobs) 157 self.claim = None 158 warn('error reading from jobserver pipe', repr(e)) 159 return 160 self.jobs += slot 161 # 162 # Add a bump for our caller's reserveration, since we're just going 163 # to sit here blocked on our child. 164 # 165 self.claim = len(self.jobs) + 1 166 167 def close(self): 168 """Return all reserved slots to Jobserver.""" 169 170 if not self.is_open: 171 return 172 173 # Return all the reserved slots. 174 if len(self.jobs): 175 os.write(self.writer, self.jobs) 176 177 self.is_open = False 178 179 def __enter__(self): 180 self.open() 181 return self 182 183 def __exit__(self, exc_type, exc_value, exc_traceback): 184 self.close() 185 186 def run(self, cmd, *args, **pwargs): 187 """ 188 Run a command setting PARALLELISM env variable to the number of 189 available job slots (claim) + 1, e.g. it will reserve claim slots 190 to do the actual build work, plus one to monitor its children. 191 192 Any %PARALLELISM% in the command's arguments is replaced by the same 193 number, or by 1 when there is no jobserver. 194 """ 195 self.open() # Ensure that self.claim is set 196 197 # We can only claim parallelism if there was a jobserver (i.e. a 198 # top-level "-jN" argument) and there were no other failures. Otherwise 199 # leave out the environment variable and let the child figure out what 200 # is best. 201 if self.claim: 202 os.environ["PARALLELISM"] = str(self.claim) 203 204 if not isinstance(cmd, str): 205 parallelism = str(self.claim or 1) 206 cmd = [arg.replace(PARALLELISM_TOKEN, parallelism) for arg in cmd] 207 208 return subprocess.call(cmd, *args, **pwargs) 209