1#!/usr/bin/env python3 2# PYTHON_ARGCOMPLETE_OKAY 3# - 4# SPDX-License-Identifier: BSD-2-Clause-FreeBSD 5# 6# Copyright (c) 2018 Alex Richardson <arichardson@FreeBSD.org> 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27# SUCH DAMAGE. 28# 29# $FreeBSD$ 30# 31 32# This script makes it easier to build on non-FreeBSD systems by bootstrapping 33# bmake and inferring required compiler variables. 34# 35# On FreeBSD you can use it the same way as just calling make: 36# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py buildworld -DWITH_FOO` 37# 38# On Linux and MacOS you will either need to set XCC/XCXX/XLD/XCPP or pass 39# --cross-bindir to specify the path to the cross-compiler bindir: 40# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py 41# --cross-bindir=/path/to/cross/compiler buildworld -DWITH_FOO TARGET=foo 42# TARGET_ARCH=bar` 43import argparse 44import os 45import shlex 46import shutil 47import subprocess 48import sys 49from pathlib import Path 50 51 52def run(cmd, **kwargs): 53 cmd = list(map(str, cmd)) # convert all Path objects to str 54 debug("Running", cmd) 55 subprocess.check_call(cmd, **kwargs) 56 57 58def bootstrap_bmake(source_root, objdir_prefix): 59 bmake_source_dir = source_root / "contrib/bmake" 60 bmake_build_dir = objdir_prefix / "bmake-build" 61 bmake_install_dir = objdir_prefix / "bmake-install" 62 bmake_binary = bmake_install_dir / "bin/bmake" 63 64 if (bmake_install_dir / "bin/bmake").exists(): 65 return bmake_binary 66 print("Bootstrapping bmake...") 67 # TODO: check if the host system bmake is new enough and use that instead 68 if not bmake_build_dir.exists(): 69 os.makedirs(str(bmake_build_dir)) 70 env = os.environ.copy() 71 global new_env_vars 72 env.update(new_env_vars) 73 74 if sys.platform.startswith("linux"): 75 # Work around the deleted file bmake/missing/sys/cdefs.h 76 # TODO: bmake should keep the compat sys/cdefs.h 77 env["CFLAGS"] = "-I{src}/tools/build/cross-build/include/common " \ 78 "-I{src}/tools/build/cross-build/include/linux " \ 79 "-D_GNU_SOURCE=1".format(src=source_root) 80 configure_args = [ 81 "--with-default-sys-path=" + str(bmake_install_dir / "share/mk"), 82 "--with-machine=amd64", # TODO? "--with-machine-arch=amd64", 83 "--without-filemon", "--prefix=" + str(bmake_install_dir)] 84 85 if Path("/bin/sh").resolve().name == "dash": 86 # Note: we have to avoid using dash as the default shell since it 87 # filters out variables containing characters such as '-' and that 88 # breaks the bmake bootstrap tests. 89 # TODO: remove this when the bootstrap tests have been fixed. 90 configure_args.append("--with-defshell=/bin/bash") 91 92 run(["sh", bmake_source_dir / "boot-strap"] + configure_args, 93 cwd=str(bmake_build_dir), env=env) 94 95 run(["sh", bmake_source_dir / "boot-strap", "op=install"] + configure_args, 96 cwd=str(bmake_build_dir)) 97 print("Finished bootstrapping bmake...") 98 return bmake_binary 99 100 101def debug(*args, **kwargs): 102 global parsed_args 103 if parsed_args.debug: 104 print(*args, **kwargs) 105 106 107def is_make_var_set(var): 108 return any( 109 x.startswith(var + "=") or x == ("-D" + var) for x in sys.argv[1:]) 110 111 112def check_required_make_env_var(varname, binary_name, bindir): 113 global new_env_vars 114 if os.getenv(varname): 115 return 116 if not bindir: 117 sys.exit("Could not infer value for $" + varname + ". Either set $" + 118 varname + " or pass --cross-bindir=/cross/compiler/dir/bin") 119 # try to infer the path to the tool 120 guess = os.path.join(bindir, binary_name) 121 if not os.path.isfile(guess): 122 sys.exit("Could not infer value for $" + varname + ": " + guess + 123 " does not exist") 124 new_env_vars[varname] = guess 125 debug("Inferred", varname, "as", guess) 126 127 128def default_cross_toolchain(): 129 # default to homebrew-installed clang on MacOS if available 130 if sys.platform.startswith("darwin"): 131 if shutil.which("brew"): 132 llvm_dir = subprocess.getoutput("brew --prefix llvm") 133 if llvm_dir and Path(llvm_dir, "bin").exists(): 134 return str(Path(llvm_dir, "bin")) 135 return None 136 137 138if __name__ == "__main__": 139 parser = argparse.ArgumentParser( 140 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 141 parser.add_argument("--host-bindir", 142 help="Directory to look for cc/c++/cpp/ld to build " 143 "host (" + sys.platform + ") binaries", 144 default="/usr/bin") 145 parser.add_argument("--cross-bindir", default=default_cross_toolchain(), 146 help="Directory to look for cc/c++/cpp/ld to build " 147 "target binaries (only needed if XCC/XCPP/XLD " 148 "are not set)") 149 parser.add_argument("--cross-compiler-type", choices=("clang", "gcc"), 150 default="clang", 151 help="Compiler type to find in --cross-bindir (only " 152 "needed if XCC/XCPP/XLD are not set)" 153 "Note: using CC is currently highly experimental") 154 parser.add_argument("--host-compiler-type", choices=("cc", "clang", "gcc"), 155 default="cc", 156 help="Compiler type to find in --host-bindir (only " 157 "needed if CC/CPP/CXX are not set). ") 158 parser.add_argument("--debug", action="store_true", 159 help="Print information on inferred env vars") 160 parser.add_argument("--clean", action="store_true", 161 help="Do a clean rebuild instead of building with " 162 "-DNO_CLEAN") 163 parser.add_argument("--no-clean", action="store_false", dest="clean", 164 help="Do a clean rebuild instead of building with " 165 "-DNO_CLEAN") 166 try: 167 import argcomplete # bash completion: 168 169 argcomplete.autocomplete(parser) 170 except ImportError: 171 pass 172 parsed_args, bmake_args = parser.parse_known_args() 173 174 MAKEOBJDIRPREFIX = os.getenv("MAKEOBJDIRPREFIX") 175 if not MAKEOBJDIRPREFIX: 176 sys.exit("MAKEOBJDIRPREFIX is not set, cannot continue!") 177 if not Path(MAKEOBJDIRPREFIX).is_dir(): 178 sys.exit( 179 "Chosen MAKEOBJDIRPREFIX=" + MAKEOBJDIRPREFIX + " doesn't exit!") 180 objdir_prefix = Path(MAKEOBJDIRPREFIX).absolute() 181 source_root = Path(__file__).absolute().parent.parent.parent 182 183 new_env_vars = {} 184 if not sys.platform.startswith("freebsd"): 185 if not is_make_var_set("TARGET") or not is_make_var_set("TARGET_ARCH"): 186 if "universe" not in sys.argv and "tinderbox" not in sys.argv: 187 sys.exit("TARGET= and TARGET_ARCH= must be set explicitly " 188 "when building on non-FreeBSD") 189 # infer values for CC/CXX/CPP 190 191 if sys.platform.startswith( 192 "linux") and parsed_args.host_compiler_type == "cc": 193 # FIXME: bsd.compiler.mk doesn't handle the output of GCC if it 194 # is /usr/bin/cc on Ubuntu since it doesn't contain the GCC string. 195 parsed_args.host_compiler_type = "gcc" 196 197 if parsed_args.host_compiler_type == "gcc": 198 default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp") 199 elif parsed_args.host_compiler_type == "clang": 200 default_cc, default_cxx, default_cpp = ( 201 "clang", "clang++", "clang-cpp") 202 else: 203 default_cc, default_cxx, default_cpp = ("cc", "c++", "cpp") 204 205 check_required_make_env_var("CC", default_cc, parsed_args.host_bindir) 206 check_required_make_env_var("CXX", default_cxx, 207 parsed_args.host_bindir) 208 check_required_make_env_var("CPP", default_cpp, 209 parsed_args.host_bindir) 210 # Using the default value for LD is fine (but not for XLD!) 211 212 use_cross_gcc = parsed_args.cross_compiler_type == "gcc" 213 # On non-FreeBSD we need to explicitly pass XCC/XLD/X_COMPILER_TYPE 214 check_required_make_env_var("XCC", "gcc" if use_cross_gcc else "clang", 215 parsed_args.cross_bindir) 216 check_required_make_env_var("XCXX", 217 "g++" if use_cross_gcc else "clang++", 218 parsed_args.cross_bindir) 219 check_required_make_env_var("XCPP", 220 "cpp" if use_cross_gcc else "clang-cpp", 221 parsed_args.cross_bindir) 222 check_required_make_env_var("XLD", "ld" if use_cross_gcc else "ld.lld", 223 parsed_args.cross_bindir) 224 225 bmake_binary = bootstrap_bmake(source_root, objdir_prefix) 226 # at -j1 cleandir+obj is unbearably slow. AUTO_OBJ helps a lot 227 debug("Adding -DWITH_AUTO_OBJ") 228 bmake_args.append("-DWITH_AUTO_OBJ") 229 if parsed_args.clean is False: 230 bmake_args.append("-DWITHOUT_CLEAN") 231 if (parsed_args.clean is None and not is_make_var_set("NO_CLEAN") 232 and not is_make_var_set("WITHOUT_CLEAN")): 233 # Avoid accidentally deleting all of the build tree and wasting lots of 234 # time cleaning directories instead of just doing a rm -rf ${.OBJDIR} 235 want_clean = input("You did not set -DNO_CLEAN/--clean/--no-clean." 236 " Did you really mean to do a clean build? y/[N] ") 237 if not want_clean.lower().startswith("y"): 238 bmake_args.append("-DNO_CLEAN") 239 240 env_cmd_str = " ".join( 241 shlex.quote(k + "=" + v) for k, v in new_env_vars.items()) 242 make_cmd_str = " ".join( 243 shlex.quote(s) for s in [str(bmake_binary)] + bmake_args) 244 debug("Running `env ", env_cmd_str, " ", make_cmd_str, "`", sep="") 245 os.environ.update(new_env_vars) 246 if parsed_args.debug: 247 input("Press enter to continue...") 248 os.chdir(str(source_root)) 249 os.execv(str(bmake_binary), [str(bmake_binary)] + bmake_args) 250