1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# pylint: disable=C0103,C0209 5# 6# 7 8""" 9Interacts with the POSIX jobserver during the Kernel build time. 10 11A "normal" jobserver task, like the one initiated by a make subrocess would do: 12 13 - open read/write file descriptors to communicate with the job server; 14 - ask for one slot by calling:: 15 16 claim = os.read(reader, 1) 17 18 - when the job finshes, call:: 19 20 os.write(writer, b"+") # os.write(writer, claim) 21 22Here, the goal is different: This script aims to get the remaining number 23of slots available, using all of them to run a command which handle tasks in 24parallel. To to that, it has a loop that ends only after there are no 25slots left. It then increments the number by one, in order to allow a 26call equivalent to ``make -j$((claim+1))``, e.g. having a parent make creating 27$claim child to do the actual work. 28 29The end goal here is to keep the total number of build tasks under the 30limit established by the initial ``make -j$n_proc`` call. 31 32See: 33 https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver 34""" 35 36import errno 37import os 38import subprocess 39import sys 40 41def warn(text, *args): 42 print(f'WARNING: {text}', *args, file = sys.stderr) 43 44class JobserverExec: 45 """ 46 Claim all slots from make using POSIX Jobserver. 47 48 The main methods here are: 49 50 - open(): reserves all slots; 51 - close(): method returns all used slots back to make; 52 - run(): executes a command setting PARALLELISM=<available slots jobs + 1>. 53 """ 54 55 def __init__(self): 56 """Initialize internal vars.""" 57 self.claim = 0 58 self.jobs = b"" 59 self.reader = None 60 self.writer = None 61 self.is_open = False 62 63 def open(self): 64 """Reserve all available slots to be claimed later on.""" 65 66 if self.is_open: 67 return 68 self.is_open = True # We only try once 69 self.claim = None 70 # 71 # Check the make flags for "--jobserver=R,W" 72 # Note that GNU Make has used --jobserver-fds and --jobserver-auth 73 # so this handles all of them. 74 # 75 flags = os.environ.get('MAKEFLAGS', '') 76 opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] 77 if not opts: 78 return 79 # 80 # Separate out the provided file descriptors 81 # 82 split_opt = opts[-1].split('=', 1) 83 if len(split_opt) != 2: 84 warn('unparseable option:', opts[-1]) 85 return 86 fds = split_opt[1] 87 # 88 # As of GNU Make 4.4, we'll be looking for a named pipe 89 # identified as fifo:path 90 # 91 if fds.startswith('fifo:'): 92 path = fds[len('fifo:'):] 93 try: 94 self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK) 95 self.writer = os.open(path, os.O_WRONLY) 96 except (OSError, IOError): 97 warn('unable to open jobserver pipe', path) 98 return 99 # 100 # Otherwise look for integer file-descriptor numbers. 101 # 102 else: 103 split_fds = fds.split(',') 104 if len(split_fds) != 2: 105 warn('malformed jobserver file descriptors:', fds) 106 return 107 try: 108 self.reader = int(split_fds[0]) 109 self.writer = int(split_fds[1]) 110 except ValueError: 111 warn('non-integer jobserver file-descriptors:', fds) 112 return 113 try: 114 # 115 # Open a private copy of reader to avoid setting nonblocking 116 # on an unexpecting process with the same reader fd. 117 # 118 self.reader = os.open(f"/proc/self/fd/{self.reader}", 119 os.O_RDONLY | os.O_NONBLOCK) 120 except (IOError, OSError) as e: 121 warn('Unable to reopen jobserver read-side pipe:', repr(e)) 122 return 123 # 124 # OK, we have the channel to the job server; read out as many jobserver 125 # slots as possible. 126 # 127 while True: 128 try: 129 slot = os.read(self.reader, 8) 130 if not slot: 131 # 132 # Something went wrong. Clear self.jobs to avoid writing 133 # weirdness back to the jobserver and give up. 134 self.jobs = b"" 135 warn("unexpected empty token from jobserver;" 136 " possible invalid '--jobserver-auth=' setting") 137 self.claim = None 138 return 139 except (OSError, IOError) as e: 140 # 141 # If there is nothing more to read then we are done. 142 # 143 if e.errno == errno.EWOULDBLOCK: 144 break 145 # 146 # Anything else says that something went weird; give back 147 # the jobs and give up. 148 # 149 if self.jobs: 150 os.write(self.writer, self.jobs) 151 self.claim = None 152 warn('error reading from jobserver pipe', repr(e)) 153 return 154 self.jobs += slot 155 # 156 # Add a bump for our caller's reserveration, since we're just going 157 # to sit here blocked on our child. 158 # 159 self.claim = len(self.jobs) + 1 160 161 def close(self): 162 """Return all reserved slots to Jobserver.""" 163 164 if not self.is_open: 165 return 166 167 # Return all the reserved slots. 168 if len(self.jobs): 169 os.write(self.writer, self.jobs) 170 171 self.is_open = False 172 173 def __enter__(self): 174 self.open() 175 return self 176 177 def __exit__(self, exc_type, exc_value, exc_traceback): 178 self.close() 179 180 def run(self, cmd, *args, **pwargs): 181 """ 182 Run a command setting PARALLELISM env variable to the number of 183 available job slots (claim) + 1, e.g. it will reserve claim slots 184 to do the actual build work, plus one to monitor its children. 185 """ 186 self.open() # Ensure that self.claim is set 187 188 # We can only claim parallelism if there was a jobserver (i.e. a 189 # top-level "-jN" argument) and there were no other failures. Otherwise 190 # leave out the environment variable and let the child figure out what 191 # is best. 192 if self.claim: 193 os.environ["PARALLELISM"] = str(self.claim) 194 195 return subprocess.call(cmd, *args, **pwargs) 196