1#!/usr/bin/env python3 2# PYTHON_ARGCOMPLETE_OKAY 3# - 4# SPDX-License-Identifier: BSD-2-Clause 5# 6# Copyright (c) 2018 Alex Richardson <arichardson@FreeBSD.org> 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27# SUCH DAMAGE. 28# 29# 30 31# This script makes it easier to build on non-FreeBSD systems by bootstrapping 32# bmake and inferring required compiler variables. 33# 34# On FreeBSD you can use it the same way as just calling make: 35# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py buildworld -DWITH_FOO` 36# 37# On Linux and MacOS you will either need to set XCC/XCXX/XLD/XCPP or pass 38# --cross-bindir to specify the path to the cross-compiler bindir: 39# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py 40# --cross-bindir=/path/to/cross/compiler buildworld -DWITH_FOO TARGET=foo 41# TARGET_ARCH=bar` 42import argparse 43import os 44import shlex 45import shutil 46import subprocess 47import sys 48from pathlib import Path 49 50 51# List of targets that are independent of TARGET/TARGET_ARCH and thus do not 52# need them to be set. Keep in the same order as Makefile documents them (if 53# they are documented). 54mach_indep_targets = [ 55 "cleanuniverse", 56 "universe", 57 "universe-toolchain", 58 "tinderbox", 59 "worlds", 60 "kernels", 61 "kernel-toolchains", 62 "targets", 63 "toolchains", 64 "makeman", 65 "sysent", 66] 67 68 69def run(cmd, **kwargs): 70 cmd = list(map(str, cmd)) # convert all Path objects to str 71 debug("Running", cmd) 72 subprocess.check_call(cmd, **kwargs) 73 74 75# Always bootstraps in order to control bmake's config to ensure compatibility 76def bootstrap_bmake(source_root, objdir_prefix): 77 bmake_source_dir = source_root / "contrib/bmake" 78 bmake_build_dir = objdir_prefix / "bmake-build" 79 bmake_install_dir = objdir_prefix / "bmake-install" 80 bmake_binary = bmake_install_dir / "bin/bmake" 81 bmake_config = bmake_install_dir / ".make-py-config" 82 83 bmake_source_version = subprocess.run([ 84 "sh", "-c", ". \"$0\"/VERSION; echo $_MAKE_VERSION", bmake_source_dir], 85 stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip() 86 try: 87 bmake_source_version = int(bmake_source_version) 88 except ValueError: 89 sys.exit("Invalid source bmake version '" + bmake_source_version + "'") 90 91 bmake_installed_version = 0 92 if bmake_binary.exists(): 93 bmake_installed_version = subprocess.run([ 94 bmake_binary, "-r", "-f", "/dev/null", "-V", "MAKE_VERSION"], 95 stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip() 96 try: 97 bmake_installed_version = int(bmake_installed_version.strip()) 98 except ValueError: 99 print("Invalid installed bmake version '" + 100 bmake_installed_version + "', treating as not present") 101 102 configure_args = [ 103 "--with-default-sys-path=.../share/mk:" + 104 str(bmake_install_dir / "share/mk"), 105 "--with-machine=amd64", # TODO? "--with-machine-arch=amd64", 106 "--without-filemon", "--prefix=" + str(bmake_install_dir)] 107 108 configure_args_str = ' '.join([shlex.quote(x) for x in configure_args]) 109 if bmake_config.exists(): 110 last_configure_args_str = bmake_config.read_text() 111 else: 112 last_configure_args_str = "" 113 114 debug("Source bmake version: " + str(bmake_source_version)) 115 debug("Installed bmake version: " + str(bmake_installed_version)) 116 debug("Configure args: " + configure_args_str) 117 debug("Last configure args: " + last_configure_args_str) 118 119 if bmake_installed_version == bmake_source_version and \ 120 configure_args_str == last_configure_args_str: 121 return bmake_binary 122 123 print("Bootstrapping bmake...") 124 if bmake_build_dir.exists(): 125 shutil.rmtree(str(bmake_build_dir)) 126 if bmake_install_dir.exists(): 127 shutil.rmtree(str(bmake_install_dir)) 128 129 os.makedirs(str(bmake_build_dir)) 130 131 env = os.environ.copy() 132 global new_env_vars 133 env.update(new_env_vars) 134 135 run(["sh", bmake_source_dir / "boot-strap"] + configure_args, 136 cwd=str(bmake_build_dir), env=env) 137 run(["sh", bmake_source_dir / "boot-strap", "op=install"] + configure_args, 138 cwd=str(bmake_build_dir)) 139 bmake_config.write_text(configure_args_str) 140 141 print("Finished bootstrapping bmake...") 142 return bmake_binary 143 144 145def debug(*args, **kwargs): 146 global parsed_args 147 if parsed_args.debug: 148 print(*args, **kwargs) 149 150 151def is_make_var_set(var): 152 return any( 153 x.startswith(var + "=") or x == ("-D" + var) for x in sys.argv[1:]) 154 155 156def check_required_make_env_var(varname, binary_name, bindir): 157 global new_env_vars 158 if os.getenv(varname): 159 return 160 if not bindir: 161 sys.exit("Could not infer value for $" + varname + ". Either set $" + 162 varname + " or pass --cross-bindir=/cross/compiler/dir/bin") 163 # try to infer the path to the tool 164 guess = os.path.join(bindir, binary_name) 165 if not os.path.isfile(guess): 166 sys.exit("Could not infer value for $" + varname + ": " + guess + 167 " does not exist") 168 new_env_vars[varname] = guess 169 debug("Inferred", varname, "as", guess) 170 global parsed_args 171 if parsed_args.debug: 172 run([guess, "--version"]) 173 174 175def check_xtool_make_env_var(varname, binary_name): 176 # Avoid calling brew --prefix on macOS if all variables are already set: 177 if os.getenv(varname): 178 return 179 global parsed_args 180 if parsed_args.cross_bindir is None: 181 parsed_args.cross_bindir = default_cross_toolchain() 182 return check_required_make_env_var(varname, binary_name, 183 parsed_args.cross_bindir) 184 185 186def default_cross_toolchain(): 187 # default to homebrew-installed clang on MacOS if available 188 if sys.platform.startswith("darwin"): 189 if shutil.which("brew"): 190 llvm_dir = subprocess.run([ 191 "brew", "--prefix", "llvm"], 192 stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip() 193 debug("Inferred LLVM dir as", llvm_dir) 194 try: 195 if llvm_dir and Path(llvm_dir.decode("utf-8"), "bin").exists(): 196 return str(Path(llvm_dir.decode("utf-8"), "bin")) 197 except OSError: 198 return None 199 return None 200 201 202if __name__ == "__main__": 203 parser = argparse.ArgumentParser( 204 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 205 parser.add_argument("--host-bindir", 206 help="Directory to look for cc/c++/cpp/ld to build " 207 "host (" + sys.platform + ") binaries", 208 default="/usr/bin") 209 parser.add_argument("--cross-bindir", default=None, 210 help="Directory to look for cc/c++/cpp/ld to build " 211 "target binaries (only needed if XCC/XCPP/XLD " 212 "are not set)") 213 parser.add_argument("--cross-compiler-type", choices=("clang", "gcc"), 214 default="clang", 215 help="Compiler type to find in --cross-bindir (only " 216 "needed if XCC/XCPP/XLD are not set)" 217 "Note: using CC is currently highly experimental") 218 parser.add_argument("--host-compiler-type", choices=("cc", "clang", "gcc"), 219 default="cc", 220 help="Compiler type to find in --host-bindir (only " 221 "needed if CC/CPP/CXX are not set). ") 222 parser.add_argument("--debug", action="store_true", 223 help="Print information on inferred env vars") 224 parser.add_argument("--bootstrap-toolchain", action="store_true", 225 help="Bootstrap the toolchain instead of using an " 226 "external one (experimental and not recommended)") 227 parser.add_argument("--clean", action="store_true", 228 help="Do a clean rebuild instead of building with " 229 "-DWITHOUT_CLEAN") 230 parser.add_argument("--no-clean", action="store_false", dest="clean", 231 help="Do a clean rebuild instead of building with " 232 "-DWITHOUT_CLEAN") 233 try: 234 import argcomplete # bash completion: 235 236 argcomplete.autocomplete(parser) 237 except ImportError: 238 pass 239 parsed_args, bmake_args = parser.parse_known_args() 240 241 MAKEOBJDIRPREFIX = os.getenv("MAKEOBJDIRPREFIX") 242 if not MAKEOBJDIRPREFIX: 243 sys.exit("MAKEOBJDIRPREFIX is not set, cannot continue!") 244 if not Path(MAKEOBJDIRPREFIX).is_dir(): 245 sys.exit( 246 "Chosen MAKEOBJDIRPREFIX=" + MAKEOBJDIRPREFIX + " doesn't exit!") 247 objdir_prefix = Path(MAKEOBJDIRPREFIX).absolute() 248 source_root = Path(__file__).absolute().parent.parent.parent 249 250 new_env_vars = {} 251 if not sys.platform.startswith("freebsd"): 252 if not is_make_var_set("TARGET") or not is_make_var_set("TARGET_ARCH"): 253 if not set(sys.argv).intersection(set(mach_indep_targets)): 254 sys.exit("TARGET= and TARGET_ARCH= must be set explicitly " 255 "when building on non-FreeBSD") 256 if not parsed_args.bootstrap_toolchain: 257 # infer values for CC/CXX/CPP 258 if parsed_args.host_compiler_type == "gcc": 259 default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp") 260 # FIXME: this should take values like `clang-9` and then look for 261 # clang-cpp-9, etc. Would alleviate the need to set the bindir on 262 # ubuntu/debian at least. 263 elif parsed_args.host_compiler_type == "clang": 264 default_cc, default_cxx, default_cpp = ( 265 "clang", "clang++", "clang-cpp") 266 else: 267 default_cc, default_cxx, default_cpp = ("cc", "c++", "cpp") 268 269 check_required_make_env_var("CC", default_cc, parsed_args.host_bindir) 270 check_required_make_env_var("CXX", default_cxx, 271 parsed_args.host_bindir) 272 check_required_make_env_var("CPP", default_cpp, 273 parsed_args.host_bindir) 274 # Using the default value for LD is fine (but not for XLD!) 275 276 # On non-FreeBSD we need to explicitly pass XCC/XLD/X_COMPILER_TYPE 277 use_cross_gcc = parsed_args.cross_compiler_type == "gcc" 278 check_xtool_make_env_var("XCC", "gcc" if use_cross_gcc else "clang") 279 check_xtool_make_env_var("XCXX", "g++" if use_cross_gcc else "clang++") 280 check_xtool_make_env_var("XCPP", 281 "cpp" if use_cross_gcc else "clang-cpp") 282 check_xtool_make_env_var("XLD", "ld" if use_cross_gcc else "ld.lld") 283 284 # We also need to set STRIPBIN if there is no working strip binary 285 # in $PATH. 286 if not shutil.which("strip"): 287 if sys.platform.startswith("darwin"): 288 # On macOS systems we have to use /usr/bin/strip. 289 sys.exit("Cannot find required tool 'strip'. Please install " 290 "the host compiler and command line tools.") 291 if parsed_args.host_compiler_type == "clang": 292 strip_binary = "llvm-strip" 293 else: 294 strip_binary = "strip" 295 check_required_make_env_var("STRIPBIN", strip_binary, 296 parsed_args.host_bindir) 297 if os.getenv("STRIPBIN") or "STRIPBIN" in new_env_vars: 298 # If we are setting STRIPBIN, we have to set XSTRIPBIN to the 299 # default if it is not set otherwise already. 300 if not os.getenv("XSTRIPBIN") and not is_make_var_set("XSTRIPBIN"): 301 # Use the bootstrapped elftoolchain strip: 302 new_env_vars["XSTRIPBIN"] = "strip" 303 304 bmake_binary = bootstrap_bmake(source_root, objdir_prefix) 305 # at -j1 cleandir+obj is unbearably slow. AUTO_OBJ helps a lot 306 debug("Adding -DWITH_AUTO_OBJ") 307 bmake_args.append("-DWITH_AUTO_OBJ") 308 if parsed_args.clean is False: 309 bmake_args.append("-DWITHOUT_CLEAN") 310 if (parsed_args.clean is None and not is_make_var_set("NO_CLEAN") 311 and not is_make_var_set("WITHOUT_CLEAN")): 312 # Avoid accidentally deleting all of the build tree and wasting lots of 313 # time cleaning directories instead of just doing a rm -rf ${.OBJDIR} 314 want_clean = input("You did not set -DWITHOUT_CLEAN/--(no-)clean." 315 " Did you really mean to do a clean build? y/[N] ") 316 if not want_clean.lower().startswith("y"): 317 bmake_args.append("-DWITHOUT_CLEAN") 318 319 env_cmd_str = " ".join( 320 shlex.quote(k + "=" + v) for k, v in new_env_vars.items()) 321 make_cmd_str = " ".join( 322 shlex.quote(s) for s in [str(bmake_binary)] + bmake_args) 323 debug("Running `env ", env_cmd_str, " ", make_cmd_str, "`", sep="") 324 os.environ.update(new_env_vars) 325 326 # Fedora defines bash function wrapper for some shell commands and this 327 # makes 'which <command>' return the function's source code instead of 328 # the binary path. Undefine it to restore the original behavior. 329 os.unsetenv("BASH_FUNC_which%%") 330 os.unsetenv("BASH_FUNC_ml%%") 331 os.unsetenv("BASH_FUNC_module%%") 332 333 os.chdir(str(source_root)) 334 os.execv(str(bmake_binary), [str(bmake_binary)] + bmake_args) 335