1#!/usr/bin/env python3 2# PYTHON_ARGCOMPLETE_OKAY 3# - 4# SPDX-License-Identifier: BSD-2-Clause 5# 6# Copyright (c) 2018 Alex Richardson <arichardson@FreeBSD.org> 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27# SUCH DAMAGE. 28# 29# 30 31# This script makes it easier to build on non-FreeBSD systems by bootstrapping 32# bmake and inferring required compiler variables. 33# 34# On FreeBSD you can use it the same way as just calling make: 35# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py buildworld -DWITH_FOO` 36# 37# On Linux and MacOS you may need to explicitly indicate the cross toolchain 38# to use. You can do this by: 39# - setting XCC/XCXX/XLD/XCPP to the paths of each tool 40# - using --cross-bindir to specify the path to the cross-compiler bindir: 41# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py 42# --cross-bindir=/path/to/cross/compiler buildworld -DWITH_FOO TARGET=foo 43# TARGET_ARCH=bar` 44# - using --cross-toolchain to specify the package containing the cross-compiler 45# (MacOS only currently): 46# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py 47# --cross-toolchain=llvm@NN buildworld -DWITH_FOO TARGET=foo 48# TARGET_ARCH=bar` 49# 50# On MacOS, this tool will search for an llvm toolchain installed via brew and 51# use it as the cross toolchain if an explicit toolchain is not specified. 52 53import argparse 54import functools 55import os 56import shlex 57import shutil 58import subprocess 59import sys 60from pathlib import Path 61 62 63# List of targets that are independent of TARGET/TARGET_ARCH and thus do not 64# need them to be set. Keep in the same order as Makefile documents them (if 65# they are documented). 66mach_indep_targets = [ 67 "cleanuniverse", 68 "universe", 69 "universe-toolchain", 70 "tinderbox", 71 "worlds", 72 "kernels", 73 "kernel-toolchains", 74 "targets", 75 "toolchains", 76 "makeman", 77 "sysent", 78] 79 80 81def run(cmd, **kwargs): 82 cmd = list(map(str, cmd)) # convert all Path objects to str 83 debug("Running", cmd) 84 subprocess.check_call(cmd, **kwargs) 85 86 87# Always bootstraps in order to control bmake's config to ensure compatibility 88def bootstrap_bmake(source_root, objdir_prefix): 89 bmake_source_dir = source_root / "contrib/bmake" 90 bmake_build_dir = objdir_prefix / "bmake-build" 91 bmake_install_dir = objdir_prefix / "bmake-install" 92 bmake_binary = bmake_install_dir / "bin/bmake" 93 bmake_config = bmake_install_dir / ".make-py-config" 94 95 bmake_source_version = subprocess.run([ 96 "sh", "-c", ". \"$0\"/VERSION; echo $_MAKE_VERSION", bmake_source_dir], 97 stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip() 98 try: 99 bmake_source_version = int(bmake_source_version) 100 except ValueError: 101 sys.exit("Invalid source bmake version '" + bmake_source_version + "'") 102 103 bmake_installed_version = 0 104 if bmake_binary.exists(): 105 bmake_installed_version = subprocess.run([ 106 bmake_binary, "-r", "-f", "/dev/null", "-V", "MAKE_VERSION"], 107 stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip() 108 try: 109 bmake_installed_version = int(bmake_installed_version.strip()) 110 except ValueError: 111 print("Invalid installed bmake version '" + 112 bmake_installed_version + "', treating as not present") 113 114 configure_args = [ 115 "--with-default-sys-path=.../share/mk:" + 116 str(bmake_install_dir / "share/mk"), 117 "--with-machine=amd64", # TODO? "--with-machine-arch=amd64", 118 "--without-filemon", "--prefix=" + str(bmake_install_dir)] 119 120 configure_args_str = ' '.join([shlex.quote(x) for x in configure_args]) 121 if bmake_config.exists(): 122 last_configure_args_str = bmake_config.read_text() 123 else: 124 last_configure_args_str = "" 125 126 debug("Source bmake version: " + str(bmake_source_version)) 127 debug("Installed bmake version: " + str(bmake_installed_version)) 128 debug("Configure args: " + configure_args_str) 129 debug("Last configure args: " + last_configure_args_str) 130 131 if bmake_installed_version == bmake_source_version and \ 132 configure_args_str == last_configure_args_str: 133 return bmake_binary 134 135 print("Bootstrapping bmake...") 136 if bmake_build_dir.exists(): 137 shutil.rmtree(str(bmake_build_dir)) 138 if bmake_install_dir.exists(): 139 shutil.rmtree(str(bmake_install_dir)) 140 141 os.makedirs(str(bmake_build_dir)) 142 143 env = os.environ.copy() 144 global new_env_vars 145 env.update(new_env_vars) 146 147 run(["sh", bmake_source_dir / "boot-strap"] + configure_args, 148 cwd=str(bmake_build_dir), env=env) 149 run(["sh", bmake_source_dir / "boot-strap", "op=install"] + configure_args, 150 cwd=str(bmake_build_dir)) 151 bmake_config.write_text(configure_args_str) 152 153 print("Finished bootstrapping bmake...") 154 return bmake_binary 155 156 157def debug(*args, **kwargs): 158 global parsed_args 159 if parsed_args.debug: 160 print(*args, **kwargs) 161 162 163def is_make_var_set(var): 164 return any( 165 x.startswith(var + "=") or x == ("-D" + var) for x in sys.argv[1:]) 166 167 168def check_required_make_env_var(varname, binary_name, bindir): 169 global new_env_vars 170 if os.getenv(varname): 171 return 172 if not bindir: 173 sys.exit("Could not infer value for $" + varname + ". Either set $" + 174 varname + " or pass --cross-bindir=/cross/compiler/dir/bin" + 175 " or --cross-toolchain=<package>") 176 # try to infer the path to the tool 177 guess = os.path.join(bindir, binary_name) 178 if not os.path.isfile(guess): 179 sys.exit("Could not infer value for $" + varname + ": " + guess + 180 " does not exist") 181 new_env_vars[varname] = guess 182 debug("Inferred", varname, "as", guess) 183 global parsed_args 184 if parsed_args.debug: 185 run([guess, "--version"]) 186 187 188def check_xtool_make_env_var(varname, binary_name): 189 # Avoid calling brew --prefix on macOS if all variables are already set: 190 if os.getenv(varname): 191 return 192 global parsed_args 193 if parsed_args.cross_bindir is None: 194 cross_bindir = cross_toolchain_bindir(binary_name, 195 parsed_args.cross_toolchain) 196 else: 197 cross_bindir = parsed_args.cross_bindir 198 return check_required_make_env_var(varname, binary_name, 199 cross_bindir) 200 201 202@functools.cache 203def brew_prefix(package: str) -> str: 204 path = subprocess.run(["brew", "--prefix", package], stdout=subprocess.PIPE, 205 stderr=subprocess.PIPE).stdout.strip() 206 debug("Inferred", package, "dir as", path) 207 return path.decode("utf-8") 208 209def binary_path(bindir: str, binary_name: str) -> "Optional[str]": 210 try: 211 if bindir and Path(bindir, "bin", binary_name).exists(): 212 return str(Path(bindir, "bin")) 213 except OSError: 214 pass 215 return None 216 217def cross_toolchain_bindir(binary_name: str, package: "Optional[str]") -> str: 218 # default to homebrew-installed clang on MacOS if available 219 if sys.platform.startswith("darwin"): 220 if shutil.which("brew"): 221 if not package: 222 package = "llvm" 223 bindir = binary_path(brew_prefix(package), binary_name) 224 if bindir: 225 return bindir 226 227 # brew installs lld as a separate package for LLVM 19 and later 228 if binary_name == "ld.lld": 229 lld_package = package.replace("llvm", "lld") 230 bindir = binary_path(brew_prefix(lld_package), binary_name) 231 if bindir: 232 return bindir 233 return None 234 235 236if __name__ == "__main__": 237 parser = argparse.ArgumentParser( 238 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 239 parser.add_argument("--host-bindir", 240 help="Directory to look for cc/c++/cpp/ld to build " 241 "host (" + sys.platform + ") binaries", 242 default="/usr/bin") 243 parser.add_argument("--cross-bindir", default=None, 244 help="Directory to look for cc/c++/cpp/ld to build " 245 "target binaries (only needed if XCC/XCPP/XLD " 246 "are not set)") 247 parser.add_argument("--cross-compiler-type", choices=("clang", "gcc"), 248 default="clang", 249 help="Compiler type to find in --cross-bindir (only " 250 "needed if XCC/XCPP/XLD are not set)" 251 "Note: using CC is currently highly experimental") 252 parser.add_argument("--cross-toolchain", default=None, 253 help="Name of package containing cc/c++/cpp/ld to build " 254 "target binaries (only needed if XCC/XCPP/XLD " 255 "are not set)") 256 parser.add_argument("--host-compiler-type", choices=("cc", "clang", "gcc"), 257 default="cc", 258 help="Compiler type to find in --host-bindir (only " 259 "needed if CC/CPP/CXX are not set). ") 260 parser.add_argument("--debug", action="store_true", 261 help="Print information on inferred env vars") 262 parser.add_argument("--bootstrap-toolchain", action="store_true", 263 help="Bootstrap the toolchain instead of using an " 264 "external one (experimental and not recommended)") 265 parser.add_argument("--clean", action="store_true", 266 help="Do a clean rebuild instead of building with " 267 "-DWITHOUT_CLEAN") 268 parser.add_argument("--no-clean", action="store_false", dest="clean", 269 help="Do a clean rebuild instead of building with " 270 "-DWITHOUT_CLEAN") 271 try: 272 import argcomplete # bash completion: 273 274 argcomplete.autocomplete(parser) 275 except ImportError: 276 pass 277 parsed_args, bmake_args = parser.parse_known_args() 278 279 MAKEOBJDIRPREFIX = os.getenv("MAKEOBJDIRPREFIX") 280 if not MAKEOBJDIRPREFIX: 281 sys.exit("MAKEOBJDIRPREFIX is not set, cannot continue!") 282 if not Path(MAKEOBJDIRPREFIX).is_dir(): 283 sys.exit( 284 "Chosen MAKEOBJDIRPREFIX=" + MAKEOBJDIRPREFIX + " doesn't exist!") 285 objdir_prefix = Path(MAKEOBJDIRPREFIX).absolute() 286 source_root = Path(__file__).absolute().parent.parent.parent 287 288 new_env_vars = {} 289 if not sys.platform.startswith("freebsd"): 290 if not is_make_var_set("TARGET") or not is_make_var_set("TARGET_ARCH"): 291 if not set(sys.argv).intersection(set(mach_indep_targets)): 292 sys.exit("TARGET= and TARGET_ARCH= must be set explicitly " 293 "when building on non-FreeBSD") 294 if not parsed_args.bootstrap_toolchain: 295 # infer values for CC/CXX/CPP 296 if parsed_args.host_compiler_type == "gcc": 297 default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp") 298 # FIXME: this should take values like `clang-9` and then look for 299 # clang-cpp-9, etc. Would alleviate the need to set the bindir on 300 # ubuntu/debian at least. 301 elif parsed_args.host_compiler_type == "clang": 302 default_cc, default_cxx, default_cpp = ( 303 "clang", "clang++", "clang-cpp") 304 else: 305 default_cc, default_cxx, default_cpp = ("cc", "c++", "cpp") 306 307 check_required_make_env_var("CC", default_cc, parsed_args.host_bindir) 308 check_required_make_env_var("CXX", default_cxx, 309 parsed_args.host_bindir) 310 check_required_make_env_var("CPP", default_cpp, 311 parsed_args.host_bindir) 312 # Using the default value for LD is fine (but not for XLD!) 313 314 # On non-FreeBSD we need to explicitly pass XCC/XLD/X_COMPILER_TYPE 315 use_cross_gcc = parsed_args.cross_compiler_type == "gcc" 316 check_xtool_make_env_var("XCC", "gcc" if use_cross_gcc else "clang") 317 check_xtool_make_env_var("XCXX", "g++" if use_cross_gcc else "clang++") 318 check_xtool_make_env_var("XCPP", 319 "cpp" if use_cross_gcc else "clang-cpp") 320 check_xtool_make_env_var("XLD", "ld" if use_cross_gcc else "ld.lld") 321 322 # We also need to set STRIPBIN if there is no working strip binary 323 # in $PATH. 324 if not shutil.which("strip"): 325 if sys.platform.startswith("darwin"): 326 # On macOS systems we have to use /usr/bin/strip. 327 sys.exit("Cannot find required tool 'strip'. Please install " 328 "the host compiler and command line tools.") 329 if parsed_args.host_compiler_type == "clang": 330 strip_binary = "llvm-strip" 331 else: 332 strip_binary = "strip" 333 check_required_make_env_var("STRIPBIN", strip_binary, 334 parsed_args.host_bindir) 335 if os.getenv("STRIPBIN") or "STRIPBIN" in new_env_vars: 336 # If we are setting STRIPBIN, we have to set XSTRIPBIN to the 337 # default if it is not set otherwise already. 338 if not os.getenv("XSTRIPBIN") and not is_make_var_set("XSTRIPBIN"): 339 # Use the bootstrapped elftoolchain strip: 340 new_env_vars["XSTRIPBIN"] = "strip" 341 342 bmake_binary = bootstrap_bmake(source_root, objdir_prefix) 343 # at -j1 cleandir+obj is unbearably slow. AUTO_OBJ helps a lot 344 debug("Adding -DWITH_AUTO_OBJ") 345 bmake_args.append("-DWITH_AUTO_OBJ") 346 if parsed_args.clean is False: 347 bmake_args.append("-DWITHOUT_CLEAN") 348 if (parsed_args.clean is None and not is_make_var_set("NO_CLEAN") 349 and not is_make_var_set("WITHOUT_CLEAN")): 350 # Avoid accidentally deleting all of the build tree and wasting lots of 351 # time cleaning directories instead of just doing a rm -rf ${.OBJDIR} 352 want_clean = input("You did not set -DWITHOUT_CLEAN/--(no-)clean." 353 " Did you really mean to do a clean build? y/[N] ") 354 if not want_clean.lower().startswith("y"): 355 bmake_args.append("-DWITHOUT_CLEAN") 356 357 env_cmd_str = " ".join( 358 shlex.quote(k + "=" + v) for k, v in new_env_vars.items()) 359 make_cmd_str = " ".join( 360 shlex.quote(s) for s in [str(bmake_binary)] + bmake_args) 361 debug("Running `env ", env_cmd_str, " ", make_cmd_str, "`", sep="") 362 os.environ.update(new_env_vars) 363 364 # Fedora defines bash function wrapper for some shell commands and this 365 # makes 'which <command>' return the function's source code instead of 366 # the binary path. Undefine it to restore the original behavior. 367 os.unsetenv("BASH_FUNC_which%%") 368 os.unsetenv("BASH_FUNC_ml%%") 369 os.unsetenv("BASH_FUNC_module%%") 370 371 os.chdir(str(source_root)) 372 os.execv(str(bmake_binary), [str(bmake_binary)] + bmake_args) 373