xref: /freebsd/tools/build/make.py (revision a03411e84728e9b267056fd31c7d1d9d1dc1b01e)
1#!/usr/bin/env python3
2# PYTHON_ARGCOMPLETE_OKAY
3# -
4# SPDX-License-Identifier: BSD-2-Clause
5#
6# Copyright (c) 2018 Alex Richardson <arichardson@FreeBSD.org>
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions
10# are met:
11# 1. Redistributions of source code must retain the above copyright
12#    notice, this list of conditions and the following disclaimer.
13# 2. Redistributions in binary form must reproduce the above copyright
14#    notice, this list of conditions and the following disclaimer in the
15#    documentation and/or other materials provided with the distribution.
16#
17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27# SUCH DAMAGE.
28#
29#
30
31# This script makes it easier to build on non-FreeBSD systems by bootstrapping
32# bmake and inferring required compiler variables.
33#
34# On FreeBSD you can use it the same way as just calling make:
35# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py buildworld -DWITH_FOO`
36#
37# On Linux and MacOS you will either need to set XCC/XCXX/XLD/XCPP or pass
38# --cross-bindir to specify the path to the cross-compiler bindir:
39# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py
40# --cross-bindir=/path/to/cross/compiler buildworld -DWITH_FOO TARGET=foo
41# TARGET_ARCH=bar`
42import argparse
43import os
44import shlex
45import shutil
46import subprocess
47import sys
48from pathlib import Path
49
50
51# List of targets that are independent of TARGET/TARGET_ARCH and thus do not
52# need them to be set. Keep in the same order as Makefile documents them (if
53# they are documented).
54mach_indep_targets = [
55    "cleanuniverse",
56    "universe",
57    "universe-toolchain",
58    "tinderbox",
59    "worlds",
60    "kernels",
61    "kernel-toolchains",
62    "targets",
63    "toolchains",
64    "makeman",
65    "sysent",
66]
67
68
69def run(cmd, **kwargs):
70    cmd = list(map(str, cmd))  # convert all Path objects to str
71    debug("Running", cmd)
72    subprocess.check_call(cmd, **kwargs)
73
74
75# Always bootstraps in order to control bmake's config to ensure compatibility
76def bootstrap_bmake(source_root, objdir_prefix):
77    bmake_source_dir = source_root / "contrib/bmake"
78    bmake_build_dir = objdir_prefix / "bmake-build"
79    bmake_install_dir = objdir_prefix / "bmake-install"
80    bmake_binary = bmake_install_dir / "bin/bmake"
81    bmake_config = bmake_install_dir / ".make-py-config"
82
83    bmake_source_version = subprocess.run([
84        "sh", "-c", ". \"$0\"/VERSION; echo $_MAKE_VERSION",
85        bmake_source_dir], capture_output=True).stdout.strip()
86    try:
87        bmake_source_version = int(bmake_source_version)
88    except ValueError:
89        sys.exit("Invalid source bmake version '" + bmake_source_version + "'")
90
91    bmake_installed_version = 0
92    if bmake_binary.exists():
93        bmake_installed_version = subprocess.run([
94            bmake_binary, "-r", "-f", "/dev/null", "-V", "MAKE_VERSION"],
95            capture_output=True).stdout.strip()
96        try:
97            bmake_installed_version = int(bmake_installed_version.strip())
98        except ValueError:
99            print("Invalid installed bmake version '" +
100                  bmake_installed_version + "', treating as not present")
101
102    configure_args = [
103        "--with-default-sys-path=.../share/mk:" +
104        str(bmake_install_dir / "share/mk"),
105        "--with-machine=amd64",  # TODO? "--with-machine-arch=amd64",
106        "--without-filemon", "--prefix=" + str(bmake_install_dir)]
107
108    configure_args_str = ' '.join([shlex.quote(x) for x in configure_args])
109    if bmake_config.exists():
110        last_configure_args_str = bmake_config.read_text()
111    else:
112        last_configure_args_str = ""
113
114    debug("Source bmake version: " + str(bmake_source_version))
115    debug("Installed bmake version: " + str(bmake_installed_version))
116    debug("Configure args: " + configure_args_str)
117    debug("Last configure args: " + last_configure_args_str)
118
119    if bmake_installed_version == bmake_source_version and \
120       configure_args_str == last_configure_args_str:
121        return bmake_binary
122
123    print("Bootstrapping bmake...")
124    if bmake_build_dir.exists():
125        shutil.rmtree(str(bmake_build_dir))
126    if bmake_install_dir.exists():
127        shutil.rmtree(str(bmake_install_dir))
128
129    os.makedirs(str(bmake_build_dir))
130
131    env = os.environ.copy()
132    global new_env_vars
133    env.update(new_env_vars)
134
135    run(["sh", bmake_source_dir / "boot-strap"] + configure_args,
136        cwd=str(bmake_build_dir), env=env)
137    run(["sh", bmake_source_dir / "boot-strap", "op=install"] + configure_args,
138        cwd=str(bmake_build_dir))
139    bmake_config.write_text(configure_args_str)
140
141    print("Finished bootstrapping bmake...")
142    return bmake_binary
143
144
145def debug(*args, **kwargs):
146    global parsed_args
147    if parsed_args.debug:
148        print(*args, **kwargs)
149
150
151def is_make_var_set(var):
152    return any(
153        x.startswith(var + "=") or x == ("-D" + var) for x in sys.argv[1:])
154
155
156def check_required_make_env_var(varname, binary_name, bindir):
157    global new_env_vars
158    if os.getenv(varname):
159        return
160    if not bindir:
161        sys.exit("Could not infer value for $" + varname + ". Either set $" +
162                 varname + " or pass --cross-bindir=/cross/compiler/dir/bin")
163    # try to infer the path to the tool
164    guess = os.path.join(bindir, binary_name)
165    if not os.path.isfile(guess):
166        sys.exit("Could not infer value for $" + varname + ": " + guess +
167                 " does not exist")
168    new_env_vars[varname] = guess
169    debug("Inferred", varname, "as", guess)
170    global parsed_args
171    if parsed_args.debug:
172        run([guess, "--version"])
173
174
175def check_xtool_make_env_var(varname, binary_name):
176    # Avoid calling brew --prefix on macOS if all variables are already set:
177    if os.getenv(varname):
178        return
179    global parsed_args
180    if parsed_args.cross_bindir is None:
181        parsed_args.cross_bindir = default_cross_toolchain()
182    return check_required_make_env_var(varname, binary_name,
183                                       parsed_args.cross_bindir)
184
185
186def default_cross_toolchain():
187    # default to homebrew-installed clang on MacOS if available
188    if sys.platform.startswith("darwin"):
189        if shutil.which("brew"):
190            llvm_dir = subprocess.run(["brew", "--prefix", "llvm"],
191                                      capture_output=True).stdout.strip()
192            debug("Inferred LLVM dir as", llvm_dir)
193            try:
194                if llvm_dir and Path(llvm_dir.decode("utf-8"), "bin").exists():
195                    return str(Path(llvm_dir.decode("utf-8"), "bin"))
196            except OSError:
197                return None
198    return None
199
200
201if __name__ == "__main__":
202    parser = argparse.ArgumentParser(
203        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
204    parser.add_argument("--host-bindir",
205                        help="Directory to look for cc/c++/cpp/ld to build "
206                             "host (" + sys.platform + ") binaries",
207                        default="/usr/bin")
208    parser.add_argument("--cross-bindir", default=None,
209                        help="Directory to look for cc/c++/cpp/ld to build "
210                             "target binaries (only needed if XCC/XCPP/XLD "
211                             "are not set)")
212    parser.add_argument("--cross-compiler-type", choices=("clang", "gcc"),
213                        default="clang",
214                        help="Compiler type to find in --cross-bindir (only "
215                             "needed if XCC/XCPP/XLD are not set)"
216                             "Note: using CC is currently highly experimental")
217    parser.add_argument("--host-compiler-type", choices=("cc", "clang", "gcc"),
218                        default="cc",
219                        help="Compiler type to find in --host-bindir (only "
220                             "needed if CC/CPP/CXX are not set). ")
221    parser.add_argument("--debug", action="store_true",
222                        help="Print information on inferred env vars")
223    parser.add_argument("--bootstrap-toolchain", action="store_true",
224                        help="Bootstrap the toolchain instead of using an "
225                             "external one (experimental and not recommended)")
226    parser.add_argument("--clean", action="store_true",
227                        help="Do a clean rebuild instead of building with "
228                             "-DWITHOUT_CLEAN")
229    parser.add_argument("--no-clean", action="store_false", dest="clean",
230                        help="Do a clean rebuild instead of building with "
231                             "-DWITHOUT_CLEAN")
232    try:
233        import argcomplete  # bash completion:
234
235        argcomplete.autocomplete(parser)
236    except ImportError:
237        pass
238    parsed_args, bmake_args = parser.parse_known_args()
239
240    MAKEOBJDIRPREFIX = os.getenv("MAKEOBJDIRPREFIX")
241    if not MAKEOBJDIRPREFIX:
242        sys.exit("MAKEOBJDIRPREFIX is not set, cannot continue!")
243    if not Path(MAKEOBJDIRPREFIX).is_dir():
244        sys.exit(
245            "Chosen MAKEOBJDIRPREFIX=" + MAKEOBJDIRPREFIX + " doesn't exit!")
246    objdir_prefix = Path(MAKEOBJDIRPREFIX).absolute()
247    source_root = Path(__file__).absolute().parent.parent.parent
248
249    new_env_vars = {}
250    if not sys.platform.startswith("freebsd"):
251        if not is_make_var_set("TARGET") or not is_make_var_set("TARGET_ARCH"):
252            if not set(sys.argv).intersection(set(mach_indep_targets)):
253                sys.exit("TARGET= and TARGET_ARCH= must be set explicitly "
254                         "when building on non-FreeBSD")
255    if not parsed_args.bootstrap_toolchain:
256        # infer values for CC/CXX/CPP
257        if parsed_args.host_compiler_type == "gcc":
258            default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp")
259        # FIXME: this should take values like `clang-9` and then look for
260        # clang-cpp-9, etc. Would alleviate the need to set the bindir on
261        # ubuntu/debian at least.
262        elif parsed_args.host_compiler_type == "clang":
263            default_cc, default_cxx, default_cpp = (
264                "clang", "clang++", "clang-cpp")
265        else:
266            default_cc, default_cxx, default_cpp = ("cc", "c++", "cpp")
267
268        check_required_make_env_var("CC", default_cc, parsed_args.host_bindir)
269        check_required_make_env_var("CXX", default_cxx,
270                                    parsed_args.host_bindir)
271        check_required_make_env_var("CPP", default_cpp,
272                                    parsed_args.host_bindir)
273        # Using the default value for LD is fine (but not for XLD!)
274
275        # On non-FreeBSD we need to explicitly pass XCC/XLD/X_COMPILER_TYPE
276        use_cross_gcc = parsed_args.cross_compiler_type == "gcc"
277        check_xtool_make_env_var("XCC", "gcc" if use_cross_gcc else "clang")
278        check_xtool_make_env_var("XCXX", "g++" if use_cross_gcc else "clang++")
279        check_xtool_make_env_var("XCPP",
280                                 "cpp" if use_cross_gcc else "clang-cpp")
281        check_xtool_make_env_var("XLD", "ld" if use_cross_gcc else "ld.lld")
282
283        # We also need to set STRIPBIN if there is no working strip binary
284        # in $PATH.
285        if not shutil.which("strip"):
286            if sys.platform.startswith("darwin"):
287                # On macOS systems we have to use /usr/bin/strip.
288                sys.exit("Cannot find required tool 'strip'. Please install "
289                         "the host compiler and command line tools.")
290            if parsed_args.host_compiler_type == "clang":
291                strip_binary = "llvm-strip"
292            else:
293                strip_binary = "strip"
294            check_required_make_env_var("STRIPBIN", strip_binary,
295                                        parsed_args.host_bindir)
296        if os.getenv("STRIPBIN") or "STRIPBIN" in new_env_vars:
297            # If we are setting STRIPBIN, we have to set XSTRIPBIN to the
298            # default if it is not set otherwise already.
299            if not os.getenv("XSTRIPBIN") and not is_make_var_set("XSTRIPBIN"):
300                # Use the bootstrapped elftoolchain strip:
301                new_env_vars["XSTRIPBIN"] = "strip"
302
303    bmake_binary = bootstrap_bmake(source_root, objdir_prefix)
304    # at -j1 cleandir+obj is unbearably slow. AUTO_OBJ helps a lot
305    debug("Adding -DWITH_AUTO_OBJ")
306    bmake_args.append("-DWITH_AUTO_OBJ")
307    if parsed_args.clean is False:
308        bmake_args.append("-DWITHOUT_CLEAN")
309    if (parsed_args.clean is None and not is_make_var_set("NO_CLEAN")
310            and not is_make_var_set("WITHOUT_CLEAN")):
311        # Avoid accidentally deleting all of the build tree and wasting lots of
312        # time cleaning directories instead of just doing a rm -rf ${.OBJDIR}
313        want_clean = input("You did not set -DWITHOUT_CLEAN/--(no-)clean."
314                           " Did you really mean to do a clean build? y/[N] ")
315        if not want_clean.lower().startswith("y"):
316            bmake_args.append("-DWITHOUT_CLEAN")
317
318    env_cmd_str = " ".join(
319        shlex.quote(k + "=" + v) for k, v in new_env_vars.items())
320    make_cmd_str = " ".join(
321        shlex.quote(s) for s in [str(bmake_binary)] + bmake_args)
322    debug("Running `env ", env_cmd_str, " ", make_cmd_str, "`", sep="")
323    os.environ.update(new_env_vars)
324
325    # Fedora defines bash function wrapper for some shell commands and this
326    # makes 'which <command>' return the function's source code instead of
327    # the binary path. Undefine it to restore the original behavior.
328    os.unsetenv("BASH_FUNC_which%%")
329    os.unsetenv("BASH_FUNC_ml%%")
330    os.unsetenv("BASH_FUNC_module%%")
331
332    os.chdir(str(source_root))
333    os.execv(str(bmake_binary), [str(bmake_binary)] + bmake_args)
334