xref: /freebsd/tools/build/make.py (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1#!/usr/bin/env python3
2# PYTHON_ARGCOMPLETE_OKAY
3# -
4# SPDX-License-Identifier: BSD-2-Clause-FreeBSD
5#
6# Copyright (c) 2018 Alex Richardson <arichardson@FreeBSD.org>
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions
10# are met:
11# 1. Redistributions of source code must retain the above copyright
12#    notice, this list of conditions and the following disclaimer.
13# 2. Redistributions in binary form must reproduce the above copyright
14#    notice, this list of conditions and the following disclaimer in the
15#    documentation and/or other materials provided with the distribution.
16#
17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27# SUCH DAMAGE.
28#
29# $FreeBSD$
30#
31
32# This script makes it easier to build on non-FreeBSD systems by bootstrapping
33# bmake and inferring required compiler variables.
34#
35# On FreeBSD you can use it the same way as just calling make:
36# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py buildworld -DWITH_FOO`
37#
38# On Linux and MacOS you will either need to set XCC/XCXX/XLD/XCPP or pass
39# --cross-bindir to specify the path to the cross-compiler bindir:
40# `MAKEOBJDIRPREFIX=~/obj ./tools/build/make.py
41# --cross-bindir=/path/to/cross/compiler buildworld -DWITH_FOO TARGET=foo
42# TARGET_ARCH=bar`
43import argparse
44import os
45import shlex
46import shutil
47import subprocess
48import sys
49from pathlib import Path
50
51
52def run(cmd, **kwargs):
53    cmd = list(map(str, cmd))  # convert all Path objects to str
54    debug("Running", cmd)
55    subprocess.check_call(cmd, **kwargs)
56
57
58def bootstrap_bmake(source_root, objdir_prefix):
59    bmake_source_dir = source_root / "contrib/bmake"
60    bmake_build_dir = objdir_prefix / "bmake-build"
61    bmake_install_dir = objdir_prefix / "bmake-install"
62    bmake_binary = bmake_install_dir / "bin/bmake"
63
64    if (bmake_install_dir / "bin/bmake").exists():
65        return bmake_binary
66    print("Bootstrapping bmake...")
67    # TODO: check if the host system bmake is new enough and use that instead
68    if not bmake_build_dir.exists():
69        os.makedirs(str(bmake_build_dir))
70    env = os.environ.copy()
71    global new_env_vars
72    env.update(new_env_vars)
73
74    if sys.platform.startswith("linux"):
75        # Work around the deleted file bmake/missing/sys/cdefs.h
76        # TODO: bmake should keep the compat sys/cdefs.h
77        env["CFLAGS"] = "-I{src}/tools/build/cross-build/include/common " \
78                        "-I{src}/tools/build/cross-build/include/linux " \
79                        "-D_GNU_SOURCE=1".format(src=source_root)
80    configure_args = [
81        "--with-default-sys-path=" + str(bmake_install_dir / "share/mk"),
82        "--with-machine=amd64",  # TODO? "--with-machine-arch=amd64",
83        "--without-filemon", "--prefix=" + str(bmake_install_dir)]
84    run(["sh", bmake_source_dir / "boot-strap"] + configure_args,
85        cwd=str(bmake_build_dir), env=env)
86
87    run(["sh", bmake_source_dir / "boot-strap", "op=install"] + configure_args,
88        cwd=str(bmake_build_dir))
89    print("Finished bootstrapping bmake...")
90    return bmake_binary
91
92
93def debug(*args, **kwargs):
94    global parsed_args
95    if parsed_args.debug:
96        print(*args, **kwargs)
97
98
99def is_make_var_set(var):
100    return any(
101        x.startswith(var + "=") or x == ("-D" + var) for x in sys.argv[1:])
102
103
104def check_required_make_env_var(varname, binary_name, bindir):
105    global new_env_vars
106    if os.getenv(varname):
107        return
108    if not bindir:
109        sys.exit("Could not infer value for $" + varname + ". Either set $" +
110                 varname + " or pass --cross-bindir=/cross/compiler/dir/bin")
111    # try to infer the path to the tool
112    guess = os.path.join(bindir, binary_name)
113    if not os.path.isfile(guess):
114        sys.exit("Could not infer value for $" + varname + ": " + guess +
115                 " does not exist")
116    new_env_vars[varname] = guess
117    debug("Inferred", varname, "as", guess)
118    global parsed_args
119    if parsed_args.debug:
120        run([guess, "--version"])
121
122
123def default_cross_toolchain():
124    # default to homebrew-installed clang on MacOS if available
125    if sys.platform.startswith("darwin"):
126        if shutil.which("brew"):
127            llvm_dir = subprocess.getoutput("brew --prefix llvm")
128            if llvm_dir and Path(llvm_dir, "bin").exists():
129                return str(Path(llvm_dir, "bin"))
130    return None
131
132
133if __name__ == "__main__":
134    parser = argparse.ArgumentParser(
135        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
136    parser.add_argument("--host-bindir",
137                        help="Directory to look for cc/c++/cpp/ld to build "
138                             "host (" + sys.platform + ") binaries",
139                        default="/usr/bin")
140    parser.add_argument("--cross-bindir", default=default_cross_toolchain(),
141                        help="Directory to look for cc/c++/cpp/ld to build "
142                             "target binaries (only needed if XCC/XCPP/XLD "
143                             "are not set)")
144    parser.add_argument("--cross-compiler-type", choices=("clang", "gcc"),
145                        default="clang",
146                        help="Compiler type to find in --cross-bindir (only "
147                             "needed if XCC/XCPP/XLD are not set)"
148                             "Note: using CC is currently highly experimental")
149    parser.add_argument("--host-compiler-type", choices=("cc", "clang", "gcc"),
150                        default="cc",
151                        help="Compiler type to find in --host-bindir (only "
152                             "needed if CC/CPP/CXX are not set). ")
153    parser.add_argument("--debug", action="store_true",
154                        help="Print information on inferred env vars")
155    parser.add_argument("--clean", action="store_true",
156                        help="Do a clean rebuild instead of building with "
157                             "-DNO_CLEAN")
158    parser.add_argument("--no-clean", action="store_false", dest="clean",
159                        help="Do a clean rebuild instead of building with "
160                             "-DNO_CLEAN")
161    try:
162        import argcomplete  # bash completion:
163
164        argcomplete.autocomplete(parser)
165    except ImportError:
166        pass
167    parsed_args, bmake_args = parser.parse_known_args()
168
169    MAKEOBJDIRPREFIX = os.getenv("MAKEOBJDIRPREFIX")
170    if not MAKEOBJDIRPREFIX:
171        sys.exit("MAKEOBJDIRPREFIX is not set, cannot continue!")
172    if not Path(MAKEOBJDIRPREFIX).is_dir():
173        sys.exit(
174            "Chosen MAKEOBJDIRPREFIX=" + MAKEOBJDIRPREFIX + " doesn't exit!")
175    objdir_prefix = Path(MAKEOBJDIRPREFIX).absolute()
176    source_root = Path(__file__).absolute().parent.parent.parent
177
178    new_env_vars = {}
179    if not sys.platform.startswith("freebsd"):
180        if not is_make_var_set("TARGET") or not is_make_var_set("TARGET_ARCH"):
181            if "universe" not in sys.argv and "tinderbox" not in sys.argv:
182                sys.exit("TARGET= and TARGET_ARCH= must be set explicitly "
183                         "when building on non-FreeBSD")
184        # infer values for CC/CXX/CPP
185
186        if sys.platform.startswith(
187                "linux") and parsed_args.host_compiler_type == "cc":
188            # FIXME: bsd.compiler.mk doesn't handle the output of GCC if it
189            #  is /usr/bin/cc on Ubuntu since it doesn't contain the GCC string.
190            parsed_args.host_compiler_type = "gcc"
191
192        if parsed_args.host_compiler_type == "gcc":
193            default_cc, default_cxx, default_cpp = ("gcc", "g++", "cpp")
194        # FIXME: this should take values like `clang-9` and then look for
195        # clang-cpp-9, etc. Would alleviate the need to set the bindir on
196        # ubuntu/debian at least.
197        elif parsed_args.host_compiler_type == "clang":
198            default_cc, default_cxx, default_cpp = (
199                "clang", "clang++", "clang-cpp")
200        else:
201            default_cc, default_cxx, default_cpp = ("cc", "c++", "cpp")
202
203        check_required_make_env_var("CC", default_cc, parsed_args.host_bindir)
204        check_required_make_env_var("CXX", default_cxx,
205                                    parsed_args.host_bindir)
206        check_required_make_env_var("CPP", default_cpp,
207                                    parsed_args.host_bindir)
208        # Using the default value for LD is fine (but not for XLD!)
209
210        # On non-FreeBSD we need to explicitly pass XCC/XLD/X_COMPILER_TYPE
211        use_cross_gcc = parsed_args.cross_compiler_type == "gcc"
212        check_required_make_env_var("XCC", "gcc" if use_cross_gcc else "clang",
213                                    parsed_args.cross_bindir)
214        check_required_make_env_var("XCXX",
215                                    "g++" if use_cross_gcc else "clang++",
216                                    parsed_args.cross_bindir)
217        check_required_make_env_var("XCPP",
218                                    "cpp" if use_cross_gcc else "clang-cpp",
219                                    parsed_args.cross_bindir)
220        check_required_make_env_var("XLD", "ld" if use_cross_gcc else "ld.lld",
221                                    parsed_args.cross_bindir)
222
223        # We also need to set STRIPBIN if there is no working strip binary
224        # in $PATH.
225        if not shutil.which("strip"):
226            if sys.platform.startswith("darwin"):
227                # On macOS systems we have to use /usr/bin/strip.
228                sys.exit("Cannot find required tool 'strip'. Please install the"
229                         " host compiler and command line tools.")
230            if parsed_args.host_compiler_type == "clang":
231                strip_binary = "llvm-strip"
232            else:
233                strip_binary = "strip"
234            check_required_make_env_var("STRIPBIN", strip_binary,
235                                        parsed_args.cross_bindir)
236        if os.getenv("STRIPBIN") or "STRIPBIN" in new_env_vars:
237            # If we are setting STRIPBIN, we have to set XSTRIPBIN to the
238            # default if it is not set otherwise already.
239            if not os.getenv("XSTRIPBIN") and not is_make_var_set("XSTRIPBIN"):
240                # Use the bootstrapped elftoolchain strip:
241                new_env_vars["XSTRIPBIN"] = "strip"
242
243    bmake_binary = bootstrap_bmake(source_root, objdir_prefix)
244    # at -j1 cleandir+obj is unbearably slow. AUTO_OBJ helps a lot
245    debug("Adding -DWITH_AUTO_OBJ")
246    bmake_args.append("-DWITH_AUTO_OBJ")
247    if parsed_args.clean is False:
248        bmake_args.append("-DWITHOUT_CLEAN")
249    if (parsed_args.clean is None and not is_make_var_set("NO_CLEAN")
250            and not is_make_var_set("WITHOUT_CLEAN")):
251        # Avoid accidentally deleting all of the build tree and wasting lots of
252        # time cleaning directories instead of just doing a rm -rf ${.OBJDIR}
253        want_clean = input("You did not set -DNO_CLEAN/--clean/--no-clean."
254                           " Did you really mean to do a  clean build? y/[N] ")
255        if not want_clean.lower().startswith("y"):
256            bmake_args.append("-DNO_CLEAN")
257
258    env_cmd_str = " ".join(
259        shlex.quote(k + "=" + v) for k, v in new_env_vars.items())
260    make_cmd_str = " ".join(
261        shlex.quote(s) for s in [str(bmake_binary)] + bmake_args)
262    debug("Running `env ", env_cmd_str, " ", make_cmd_str, "`", sep="")
263    os.environ.update(new_env_vars)
264    os.chdir(str(source_root))
265    os.execv(str(bmake_binary), [str(bmake_binary)] + bmake_args)
266